{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1661}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4196.449, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5613c7176f00] top block unavailable for requested intra mode -1 [h264 @ 0x5613c7176f00] error while decoding MB 37 0, bytestream 6709 [h264 @ 0x5613c726a540] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5613c726a540] error while decoding MB 0 4, bytestream 527 [h264 @ 0x5613c7176f00] error while decoding MB 40 34, bytestream -5 [h264 @ 0x5613c71cde40] left block unavailable for requested intra mode [h264 @ 0x5613c71cde40] error while decoding MB 0 44, bytestream 8298", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 174, "h264_decode_error_count": 247, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:03:45+09:00", "last_error_summary": "[h264 @ 0x5613c7176f00] top block unavailable for requested intra mode -1 [h264 @ 0x5613c7176f00] error while decoding MB 37 0, bytestream 6709 [h264 @ 0x5613c726a540] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5613c726a540] error while decoding MB 0 4, bytestream 527 [h264 @ 0x5613c7176f00] error while decoding MB 40 34, bytestream -5 [h264 @ 0x5613c71cde40] left block unavailable for requested intra mode [h264 @ 0x5613c71cde40] error while decoding MB 0 44, bytestream 8298", "last_exit_at": "2026-07-01T23:52:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 686.431, "last_started_at": "2026-07-01T23:52:18+09:00", "pid": 30361, "process": {"exists": false, "pid": 30361}, "restart_count": 198, "restart_count_per_hour": 5, "rtsp_timeout_count": 44, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 199}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.749, "sample_count": 2674, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 199342, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:03:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.071, "last_frame_pushed_at": "2026-07-02T00:03:45+09:00", "last_frame_read_age_sec": 0.058, "last_frame_read_at": "2026-07-02T00:03:45+09:00", "mjpeg": {"age_sec": 0.054, "exists": true, "mtime": "2026-07-02T00:03:45.125647+09:00", "mtime_epoch": 1782918225.125647, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 117362}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:03:45+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 962.023, "name": "celery", "pid": 237, "rss_mb": 956.156, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 90, "pushed_frames": 90, "read_frames": 91, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.001, "pushed": 3.001, "read": 3.035}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.982, "count": 90, "max_ms": 5.042, "p95_ms": 3.639}, "gstreamer_push": {"avg_ms": 1.524, "count": 90, "max_ms": 6.261, "p95_ms": 3.55}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.392, "count": 90, "max_ms": 77.378, "p95_ms": 75.803}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.944, "count": 90, "max_ms": 89.754, "p95_ms": 85.582}, "yolo11_640_fire_11class": {"avg_ms": 70.425, "count": 90, "max_ms": 83.719, "p95_ms": 80.413}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.925, "count": 90, "max_ms": 90.574, "p95_ms": 87.283}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.741, "count": 90, "max_ms": 141.541, "p95_ms": 136.506}, "read_interval": {"avg_ms": 330.125, "count": 91, "max_ms": 19737.063, "p95_ms": 151.945}, "triton_wait": {"avg_ms": 78.732, "count": 90, "max_ms": 90.585, "p95_ms": 87.309}, "visualize": {"avg_ms": 3.921, "count": 90, "max_ms": 6.508, "p95_ms": 5.255}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.986, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1661}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4195.715, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T00:03:45+09:00", "last_error_summary": "[h264 @ 0x5613c7176f00] top block unavailable for requested intra mode -1 [h264 @ 0x5613c7176f00] error while decoding MB 37 0, bytestream 6709 [h264 @ 0x5613c726a540] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5613c726a540] error while decoding MB 0 4, bytestream 527 [h264 @ 0x5613c7176f00] error while decoding MB 40 34, bytestream -5 [h264 @ 0x5613c71cde40] left block unavailable for requested intra mode [h264 @ 0x5613c71cde40] error while decoding MB 0 44, bytestream 8298", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 174, "h264_decode_error_count": 247, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:03:45+09:00", "last_error_summary": "[h264 @ 0x5613c7176f00] top block unavailable for requested intra mode -1 [h264 @ 0x5613c7176f00] error while decoding MB 37 0, bytestream 6709 [h264 @ 0x5613c726a540] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5613c726a540] error while decoding MB 0 4, bytestream 527 [h264 @ 0x5613c7176f00] error while decoding MB 40 34, bytestream -5 [h264 @ 0x5613c71cde40] left block unavailable for requested intra mode [h264 @ 0x5613c71cde40] error while decoding MB 0 44, bytestream 8298", "last_exit_at": "2026-07-02T00:03:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 686.485, "last_started_at": "2026-07-01T23:52:18+09:00", "pid": 30361, "process": {"exists": false, "pid": 30361}, "restart_count": 198, "restart_count_per_hour": 5, "rtsp_timeout_count": 44, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 199}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.749, "sample_count": 2674, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 199343, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:03:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-02T00:03:45+09:00", "last_frame_read_age_sec": 0.112, "last_frame_read_at": "2026-07-02T00:03:45+09:00", "mjpeg": {"age_sec": 0.108, "exists": true, "mtime": "2026-07-02T00:03:45.125647+09:00", "mtime_epoch": 1782918225.125647, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 117362}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:03:45+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 962.023, "name": "celery", "pid": 237, "rss_mb": 956.156, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 91, "pushed_frames": 91, "read_frames": 90, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.038, "pushed": 3.038, "read": 3.005}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.976, "count": 91, "max_ms": 5.042, "p95_ms": 3.639}, "gstreamer_push": {"avg_ms": 1.523, "count": 91, "max_ms": 6.261, "p95_ms": 3.55}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.349, "count": 91, "max_ms": 77.378, "p95_ms": 75.803}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.925, "count": 91, "max_ms": 89.754, "p95_ms": 85.582}, "yolo11_640_fire_11class": {"avg_ms": 70.501, "count": 91, "max_ms": 83.719, "p95_ms": 80.413}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.025, "count": 91, "max_ms": 90.574, "p95_ms": 87.283}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.724, "count": 91, "max_ms": 141.541, "p95_ms": 136.506}, "read_interval": {"avg_ms": 332.522, "count": 90, "max_ms": 19737.063, "p95_ms": 151.945}, "triton_wait": {"avg_ms": 78.822, "count": 91, "max_ms": 90.585, "p95_ms": 87.309}, "visualize": {"avg_ms": 3.918, "count": 91, "max_ms": 6.508, "p95_ms": 5.255}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.949, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1661}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4195.832, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 174, "h264_decode_error_count": 247, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:03:45+09:00", "last_error_summary": "[h264 @ 0x5613c7176f00] top block unavailable for requested intra mode -1 [h264 @ 0x5613c7176f00] error while decoding MB 37 0, bytestream 6709 [h264 @ 0x5613c726a540] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5613c726a540] error while decoding MB 0 4, bytestream 527 [h264 @ 0x5613c7176f00] error while decoding MB 40 34, bytestream -5 [h264 @ 0x5613c71cde40] left block unavailable for requested intra mode [h264 @ 0x5613c71cde40] error while decoding MB 0 44, bytestream 8298", "last_exit_at": "2026-07-02T00:03:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 686.492, "last_started_at": "2026-07-01T23:52:18+09:00", "pid": 30361, "process": {"exists": false, "pid": 30361}, "restart_count": 198, "restart_count_per_hour": 5, "rtsp_timeout_count": 44, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 199}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.749, "sample_count": 2674, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 199343, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:03:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.02, "last_frame_pushed_at": "2026-07-02T00:03:45+09:00", "last_frame_read_age_sec": 0.12, "last_frame_read_at": "2026-07-02T00:03:45+09:00", "mjpeg": {"age_sec": 0.003, "exists": true, "mtime": "2026-07-02T00:03:45.237647+09:00", "mtime_epoch": 1782918225.2376468, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 114569}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:03:45+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 962.023, "name": "celery", "pid": 237, "rss_mb": 956.156, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 91, "pushed_frames": 91, "read_frames": 90, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.038, "pushed": 3.038, "read": 3.004}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.976, "count": 91, "max_ms": 5.042, "p95_ms": 3.639}, "gstreamer_push": {"avg_ms": 1.523, "count": 91, "max_ms": 6.261, "p95_ms": 3.55}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.349, "count": 91, "max_ms": 77.378, "p95_ms": 75.803}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.925, "count": 91, "max_ms": 89.754, "p95_ms": 85.582}, "yolo11_640_fire_11class": {"avg_ms": 70.501, "count": 91, "max_ms": 83.719, "p95_ms": 80.413}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.025, "count": 91, "max_ms": 90.574, "p95_ms": 87.283}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.724, "count": 91, "max_ms": 141.541, "p95_ms": 136.506}, "read_interval": {"avg_ms": 332.522, "count": 90, "max_ms": 19737.063, "p95_ms": 151.945}, "triton_wait": {"avg_ms": 78.822, "count": 91, "max_ms": 90.585, "p95_ms": 87.309}, "visualize": {"avg_ms": 3.918, "count": 91, "max_ms": 6.508, "p95_ms": 5.255}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.956, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1661}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4298.355, "limit_mb": 8796093022207.996}, "detail": {"pid": 30620, "restart_count": 199, "start_count": 200}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 174, "h264_decode_error_count": 247, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:03:45+09:00", "last_error_summary": "[h264 @ 0x5613c7176f00] top block unavailable for requested intra mode -1 [h264 @ 0x5613c7176f00] error while decoding MB 37 0, bytestream 6709 [h264 @ 0x5613c726a540] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5613c726a540] error while decoding MB 0 4, bytestream 527 [h264 @ 0x5613c7176f00] error while decoding MB 40 34, bytestream -5 [h264 @ 0x5613c71cde40] left block unavailable for requested intra mode [h264 @ 0x5613c71cde40] error while decoding MB 0 44, bytestream 8298", "last_exit_at": "2026-07-02T00:03:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T00:03:47+09:00", "pid": 30620, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.883, "name": "ffmpeg", "pid": 30620, "rss_mb": 138.883, "state": "S (sleeping)", "threads": 106}, "restart_count": 199, "restart_count_per_hour": 6, "rtsp_timeout_count": 44, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 200}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.749, "sample_count": 2625, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 199343, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:03:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.636, "last_frame_pushed_at": "2026-07-02T00:03:45+09:00", "last_frame_read_age_sec": 4.736, "last_frame_read_at": "2026-07-02T00:03:45+09:00", "mjpeg": {"age_sec": 4.619, "exists": true, "mtime": "2026-07-02T00:03:45.237647+09:00", "mtime_epoch": 1782918225.2376468, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 114569}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:03:49+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 962.023, "name": "celery", "pid": 237, "rss_mb": 956.156, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 49, "pushed_frames": 49, "read_frames": 48, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.637, "pushed": 1.637, "read": 1.604}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.445, "count": 49, "max_ms": 5.042, "p95_ms": 4.131}, "gstreamer_push": {"avg_ms": 1.457, "count": 49, "max_ms": 6.261, "p95_ms": 3.153}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.378, "count": 49, "max_ms": 75.066, "p95_ms": 73.373}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.271, "count": 49, "max_ms": 89.754, "p95_ms": 86.461}, "yolo11_640_fire_11class": {"avg_ms": 67.072, "count": 49, "max_ms": 83.7, "p95_ms": 77.564}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.358, "count": 49, "max_ms": 90.574, "p95_ms": 87.44}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.093, "count": 49, "max_ms": 140.28, "p95_ms": 138.739}, "read_interval": {"avg_ms": 527.242, "count": 48, "max_ms": 19737.063, "p95_ms": 152.542}, "triton_wait": {"avg_ms": 75.337, "count": 49, "max_ms": 90.585, "p95_ms": 87.451}, "visualize": {"avg_ms": 4.394, "count": 49, "max_ms": 6.508, "p95_ms": 5.305}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.929, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1690}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4201.602, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55733d3e8480] error while decoding MB 69 12, bytestream -12 [h264 @ 0x55733cac1b40] left block unavailable for requested intra mode [h264 @ 0x55733cac1b40] error while decoding MB 0 6, bytestream 6556 [in#0/rtsp @ 0x55733ca3be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55733ca3be00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 175, "h264_decode_error_count": 249, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:10:11+09:00", "last_error_summary": "[h264 @ 0x55733d3e8480] error while decoding MB 69 12, bytestream -12 [h264 @ 0x55733cac1b40] left block unavailable for requested intra mode [h264 @ 0x55733cac1b40] error while decoding MB 0 6, bytestream 6556 [in#0/rtsp @ 0x55733ca3be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55733ca3be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T00:03:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 383.308, "last_started_at": "2026-07-02T00:03:47+09:00", "pid": 30620, "process": {"exists": false, "pid": 30620}, "restart_count": 199, "restart_count_per_hour": 5, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 200}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 134.07, "sample_count": 2681, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 202823, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:10:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.069, "last_frame_pushed_at": "2026-07-02T00:10:11+09:00", "last_frame_read_age_sec": 0.056, "last_frame_read_at": "2026-07-02T00:10:11+09:00", "mjpeg": {"age_sec": 0.053, "exists": true, "mtime": "2026-07-02T00:10:11.110998+09:00", "mtime_epoch": 1782918611.110998, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 104122}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:10:11+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 963.758, "name": "celery", "pid": 237, "rss_mb": 958.484, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 104, "pushed_frames": 104, "read_frames": 104, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.474, "pushed": 3.474, "read": 3.474}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.005, "count": 104, "max_ms": 4.351, "p95_ms": 3.633}, "gstreamer_push": {"avg_ms": 1.528, "count": 104, "max_ms": 6.059, "p95_ms": 3.596}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.216, "count": 104, "max_ms": 79.991, "p95_ms": 75.56}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.738, "count": 104, "max_ms": 91.488, "p95_ms": 85.802}, "yolo11_640_fire_11class": {"avg_ms": 62.319, "count": 104, "max_ms": 80.633, "p95_ms": 77.23}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.622, "count": 104, "max_ms": 90.56, "p95_ms": 89.18}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.993, "count": 104, "max_ms": 140.386, "p95_ms": 137.235}, "read_interval": {"avg_ms": 287.929, "count": 104, "max_ms": 10132.878, "p95_ms": 151.435}, "triton_wait": {"avg_ms": 69.689, "count": 104, "max_ms": 91.5, "p95_ms": 89.192}, "visualize": {"avg_ms": 4.032, "count": 104, "max_ms": 5.675, "p95_ms": 5.169}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.935, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1690}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4201.172, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T00:10:11+09:00", "last_error_summary": "[h264 @ 0x55733d3e8480] error while decoding MB 69 12, bytestream -12 [h264 @ 0x55733cac1b40] left block unavailable for requested intra mode [h264 @ 0x55733cac1b40] error while decoding MB 0 6, bytestream 6556 [in#0/rtsp @ 0x55733ca3be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55733ca3be00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 175, "h264_decode_error_count": 249, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:10:11+09:00", "last_error_summary": "[h264 @ 0x55733d3e8480] error while decoding MB 69 12, bytestream -12 [h264 @ 0x55733cac1b40] left block unavailable for requested intra mode [h264 @ 0x55733cac1b40] error while decoding MB 0 6, bytestream 6556 [in#0/rtsp @ 0x55733ca3be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55733ca3be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T00:10:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 383.332, "last_started_at": "2026-07-02T00:03:47+09:00", "pid": 30620, "process": {"exists": false, "pid": 30620}, "restart_count": 199, "restart_count_per_hour": 5, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 200}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 134.07, "sample_count": 2682, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 202824, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:10:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T00:10:11+09:00", "last_frame_read_age_sec": 0.08, "last_frame_read_at": "2026-07-02T00:10:11+09:00", "mjpeg": {"age_sec": 0.077, "exists": true, "mtime": "2026-07-02T00:10:11.110998+09:00", "mtime_epoch": 1782918611.110998, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 104122}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:10:11+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 963.758, "name": "celery", "pid": 237, "rss_mb": 958.484, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 105, "pushed_frames": 105, "read_frames": 104, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.505, "pushed": 3.505, "read": 3.471}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.999, "count": 105, "max_ms": 4.351, "p95_ms": 3.633}, "gstreamer_push": {"avg_ms": 1.526, "count": 105, "max_ms": 6.059, "p95_ms": 3.596}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.144, "count": 105, "max_ms": 79.991, "p95_ms": 75.56}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.666, "count": 105, "max_ms": 91.488, "p95_ms": 85.802}, "yolo11_640_fire_11class": {"avg_ms": 62.232, "count": 105, "max_ms": 80.633, "p95_ms": 77.23}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.491, "count": 105, "max_ms": 90.56, "p95_ms": 89.18}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.834, "count": 105, "max_ms": 140.386, "p95_ms": 137.235}, "read_interval": {"avg_ms": 287.929, "count": 104, "max_ms": 10132.878, "p95_ms": 151.435}, "triton_wait": {"avg_ms": 69.599, "count": 105, "max_ms": 91.5, "p95_ms": 89.192}, "visualize": {"avg_ms": 4.03, "count": 105, "max_ms": 5.675, "p95_ms": 5.169}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.959, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1690}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4200.941, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 175, "h264_decode_error_count": 249, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:10:11+09:00", "last_error_summary": "[h264 @ 0x55733d3e8480] error while decoding MB 69 12, bytestream -12 [h264 @ 0x55733cac1b40] left block unavailable for requested intra mode [h264 @ 0x55733cac1b40] error while decoding MB 0 6, bytestream 6556 [in#0/rtsp @ 0x55733ca3be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55733ca3be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T00:10:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 383.34, "last_started_at": "2026-07-02T00:03:47+09:00", "pid": 30620, "process": {"exists": false, "pid": 30620}, "restart_count": 199, "restart_count_per_hour": 5, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 200}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 134.07, "sample_count": 2682, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 202824, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:10:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-02T00:10:11+09:00", "last_frame_read_age_sec": 0.088, "last_frame_read_at": "2026-07-02T00:10:11+09:00", "mjpeg": {"age_sec": 0.085, "exists": true, "mtime": "2026-07-02T00:10:11.110998+09:00", "mtime_epoch": 1782918611.110998, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 104122}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:10:11+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 963.758, "name": "celery", "pid": 237, "rss_mb": 958.484, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 105, "pushed_frames": 105, "read_frames": 104, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.504, "pushed": 3.504, "read": 3.471}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.999, "count": 105, "max_ms": 4.351, "p95_ms": 3.633}, "gstreamer_push": {"avg_ms": 1.526, "count": 105, "max_ms": 6.059, "p95_ms": 3.596}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.144, "count": 105, "max_ms": 79.991, "p95_ms": 75.56}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.666, "count": 105, "max_ms": 91.488, "p95_ms": 85.802}, "yolo11_640_fire_11class": {"avg_ms": 62.232, "count": 105, "max_ms": 80.633, "p95_ms": 77.23}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.491, "count": 105, "max_ms": 90.56, "p95_ms": 89.18}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.834, "count": 105, "max_ms": 140.386, "p95_ms": 137.235}, "read_interval": {"avg_ms": 287.929, "count": 104, "max_ms": 10132.878, "p95_ms": 151.435}, "triton_wait": {"avg_ms": 69.599, "count": 105, "max_ms": 91.5, "p95_ms": 89.192}, "visualize": {"avg_ms": 4.03, "count": 105, "max_ms": 5.675, "p95_ms": 5.169}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.967, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1690}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4311.363, "limit_mb": 8796093022207.996}, "detail": {"pid": 30727, "restart_count": 200, "start_count": 201}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 175, "h264_decode_error_count": 249, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:10:11+09:00", "last_error_summary": "[h264 @ 0x55733d3e8480] error while decoding MB 69 12, bytestream -12 [h264 @ 0x55733cac1b40] left block unavailable for requested intra mode [h264 @ 0x55733cac1b40] error while decoding MB 0 6, bytestream 6556 [in#0/rtsp @ 0x55733ca3be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55733ca3be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T00:10:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T00:10:14+09:00", "pid": 30727, "process": {"exists": true, "fd_count": 5, "hwm_mb": 142.582, "name": "ffmpeg", "pid": 30727, "rss_mb": 142.582, "state": "S (sleeping)", "threads": 106}, "restart_count": 200, "restart_count_per_hour": 6, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 201}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 134.07, "sample_count": 2636, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 202824, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:10:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.991, "last_frame_pushed_at": "2026-07-02T00:10:11+09:00", "last_frame_read_age_sec": 5.064, "last_frame_read_at": "2026-07-02T00:10:11+09:00", "mjpeg": {"age_sec": 4.973, "exists": true, "mtime": "2026-07-02T00:10:11.198998+09:00", "mtime_epoch": 1782918611.1989985, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 103475}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:10:16+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 963.758, "name": "celery", "pid": 237, "rss_mb": 958.484, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 53, "pushed_frames": 53, "read_frames": 52, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.769, "pushed": 1.769, "read": 1.736}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.014, "count": 53, "max_ms": 3.903, "p95_ms": 3.632}, "gstreamer_push": {"avg_ms": 1.39, "count": 53, "max_ms": 3.529, "p95_ms": 1.487}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.863, "count": 53, "max_ms": 79.991, "p95_ms": 76.833}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.799, "count": 53, "max_ms": 87.642, "p95_ms": 85.988}, "yolo11_640_fire_11class": {"avg_ms": 67.788, "count": 53, "max_ms": 80.633, "p95_ms": 79.098}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.104, "count": 53, "max_ms": 90.526, "p95_ms": 89.338}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.556, "count": 53, "max_ms": 140.386, "p95_ms": 137.653}, "read_interval": {"avg_ms": 480.383, "count": 52, "max_ms": 10132.878, "p95_ms": 153.767}, "triton_wait": {"avg_ms": 77.731, "count": 53, "max_ms": 90.539, "p95_ms": 89.352}, "visualize": {"avg_ms": 4.067, "count": 53, "max_ms": 5.263, "p95_ms": 5.157}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.953, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1725}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4205.812, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55c914327400] error while decoding MB 74 21, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 176, "h264_decode_error_count": 250, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:18:12+09:00", "last_error_summary": "[h264 @ 0x55c914327400] error while decoding MB 74 21, bytestream -5", "last_exit_at": "2026-07-02T00:10:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 478.479, "last_started_at": "2026-07-02T00:10:14+09:00", "pid": 30727, "process": {"exists": false, "pid": 30727}, "restart_count": 200, "restart_count_per_hour": 5, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 201}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 138.407, "sample_count": 2643, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 207077, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:18:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.069, "last_frame_pushed_at": "2026-07-02T00:18:12+09:00", "last_frame_read_age_sec": 0.056, "last_frame_read_at": "2026-07-02T00:18:12+09:00", "mjpeg": {"age_sec": 0.051, "exists": true, "mtime": "2026-07-02T00:18:12.599922+09:00", "mtime_epoch": 1782919092.5999222, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 107450}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:18:12+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 966.238, "name": "celery", "pid": 237, "rss_mb": 963.707, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 230, "pushed_frames": 230, "read_frames": 230, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.677, "pushed": 7.677, "read": 7.677}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.467, "count": 229, "max_ms": 5.508, "p95_ms": 4.176}, "gstreamer_push": {"avg_ms": 1.31, "count": 230, "max_ms": 6.167, "p95_ms": 1.478}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.844, "count": 229, "max_ms": 77.807, "p95_ms": 69.584}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.614, "count": 229, "max_ms": 91.018, "p95_ms": 86.865}, "yolo11_640_fire_11class": {"avg_ms": 62.017, "count": 229, "max_ms": 83.274, "p95_ms": 76.075}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.109, "count": 229, "max_ms": 90.686, "p95_ms": 87.163}}, "truncated_count": 0}, "processing_total": {"avg_ms": 117.074, "count": 230, "max_ms": 146.168, "p95_ms": 139.519}, "read_interval": {"avg_ms": 130.578, "count": 230, "max_ms": 159.914, "p95_ms": 153.431}, "triton_wait": {"avg_ms": 70.41, "count": 229, "max_ms": 91.03, "p95_ms": 87.84}, "visualize": {"avg_ms": 5.026, "count": 229, "max_ms": 6.753, "p95_ms": 5.558}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.961, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1725}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4199.121, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T00:18:12+09:00", "last_error_summary": "[h264 @ 0x55c914327400] error while decoding MB 74 21, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 176, "h264_decode_error_count": 250, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:18:12+09:00", "last_error_summary": "[h264 @ 0x55c914327400] error while decoding MB 74 21, bytestream -5", "last_exit_at": "2026-07-02T00:18:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 478.548, "last_started_at": "2026-07-02T00:10:14+09:00", "pid": 30727, "process": {"exists": false, "pid": 30727}, "restart_count": 200, "restart_count_per_hour": 5, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 201}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 138.407, "sample_count": 2644, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 207078, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:18:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T00:18:12+09:00", "last_frame_read_age_sec": 0.125, "last_frame_read_at": "2026-07-02T00:18:12+09:00", "mjpeg": {"age_sec": 0.12, "exists": true, "mtime": "2026-07-02T00:18:12.599922+09:00", "mtime_epoch": 1782919092.5999222, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 107450}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:18:12+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 966.238, "name": "celery", "pid": 237, "rss_mb": 957.578, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 230, "pushed_frames": 230, "read_frames": 229, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.692, "pushed": 7.692, "read": 7.658}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.47, "count": 230, "max_ms": 5.508, "p95_ms": 4.176}, "gstreamer_push": {"avg_ms": 1.316, "count": 230, "max_ms": 6.167, "p95_ms": 1.804}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.788, "count": 230, "max_ms": 77.807, "p95_ms": 69.584}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.574, "count": 230, "max_ms": 91.018, "p95_ms": 86.865}, "yolo11_640_fire_11class": {"avg_ms": 61.951, "count": 230, "max_ms": 83.274, "p95_ms": 76.075}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.04, "count": 230, "max_ms": 90.686, "p95_ms": 87.163}}, "truncated_count": 0}, "processing_total": {"avg_ms": 117.022, "count": 230, "max_ms": 146.168, "p95_ms": 139.519}, "read_interval": {"avg_ms": 130.531, "count": 229, "max_ms": 159.914, "p95_ms": 153.431}, "triton_wait": {"avg_ms": 70.367, "count": 230, "max_ms": 91.03, "p95_ms": 87.84}, "visualize": {"avg_ms": 5.033, "count": 230, "max_ms": 6.753, "p95_ms": 5.566}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.903, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1725}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4199.113, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 176, "h264_decode_error_count": 250, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:18:12+09:00", "last_error_summary": "[h264 @ 0x55c914327400] error while decoding MB 74 21, bytestream -5", "last_exit_at": "2026-07-02T00:18:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 478.554, "last_started_at": "2026-07-02T00:10:14+09:00", "pid": 30727, "process": {"exists": false, "pid": 30727}, "restart_count": 200, "restart_count_per_hour": 5, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 201}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 138.407, "sample_count": 2644, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 207078, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:18:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-02T00:18:12+09:00", "last_frame_read_age_sec": 0.131, "last_frame_read_at": "2026-07-02T00:18:12+09:00", "mjpeg": {"age_sec": 0.126, "exists": true, "mtime": "2026-07-02T00:18:12.599922+09:00", "mtime_epoch": 1782919092.5999222, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 107450}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:18:12+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 966.238, "name": "celery", "pid": 237, "rss_mb": 957.578, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 230, "pushed_frames": 230, "read_frames": 229, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.69, "pushed": 7.69, "read": 7.657}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.47, "count": 230, "max_ms": 5.508, "p95_ms": 4.176}, "gstreamer_push": {"avg_ms": 1.316, "count": 230, "max_ms": 6.167, "p95_ms": 1.804}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.788, "count": 230, "max_ms": 77.807, "p95_ms": 69.584}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.574, "count": 230, "max_ms": 91.018, "p95_ms": 86.865}, "yolo11_640_fire_11class": {"avg_ms": 61.951, "count": 230, "max_ms": 83.274, "p95_ms": 76.075}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.04, "count": 230, "max_ms": 90.686, "p95_ms": 87.163}}, "truncated_count": 0}, "processing_total": {"avg_ms": 117.022, "count": 230, "max_ms": 146.168, "p95_ms": 139.519}, "read_interval": {"avg_ms": 130.531, "count": 229, "max_ms": 159.914, "p95_ms": 153.431}, "triton_wait": {"avg_ms": 70.367, "count": 230, "max_ms": 91.03, "p95_ms": 87.84}, "visualize": {"avg_ms": 5.033, "count": 230, "max_ms": 6.753, "p95_ms": 5.566}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.908, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1725}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4307.074, "limit_mb": 8796093022207.996}, "detail": {"pid": 30834, "restart_count": 201, "start_count": 202}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 176, "h264_decode_error_count": 250, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:18:12+09:00", "last_error_summary": "[h264 @ 0x55c914327400] error while decoding MB 74 21, bytestream -5", "last_exit_at": "2026-07-02T00:18:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T00:18:15+09:00", "pid": 30834, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.223, "name": "ffmpeg", "pid": 30834, "rss_mb": 139.223, "state": "S (sleeping)", "threads": 106}, "restart_count": 201, "restart_count_per_hour": 6, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 202}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 138.42, "sample_count": 2594, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 207078, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:18:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.92, "last_frame_pushed_at": "2026-07-02T00:18:12+09:00", "last_frame_read_age_sec": 5.04, "last_frame_read_at": "2026-07-02T00:18:12+09:00", "mjpeg": {"age_sec": 4.903, "exists": true, "mtime": "2026-07-02T00:18:12.731924+09:00", "mtime_epoch": 1782919092.7319238, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 106970}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:18:17+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 966.238, "name": "celery", "pid": 237, "rss_mb": 957.578, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 193, "pushed_frames": 193, "read_frames": 192, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.45, "pushed": 6.45, "read": 6.416}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.455, "count": 193, "max_ms": 5.508, "p95_ms": 4.29}, "gstreamer_push": {"avg_ms": 1.324, "count": 193, "max_ms": 6.167, "p95_ms": 1.804}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.787, "count": 193, "max_ms": 77.807, "p95_ms": 70.534}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.405, "count": 193, "max_ms": 91.018, "p95_ms": 87.068}, "yolo11_640_fire_11class": {"avg_ms": 61.817, "count": 193, "max_ms": 83.274, "p95_ms": 76.657}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.893, "count": 193, "max_ms": 90.686, "p95_ms": 87.487}}, "truncated_count": 0}, "processing_total": {"avg_ms": 116.684, "count": 193, "max_ms": 146.168, "p95_ms": 140.257}, "read_interval": {"avg_ms": 130.222, "count": 192, "max_ms": 159.914, "p95_ms": 154.6}, "triton_wait": {"avg_ms": 70.196, "count": 193, "max_ms": 91.03, "p95_ms": 87.87}, "visualize": {"avg_ms": 5.026, "count": 193, "max_ms": 6.753, "p95_ms": 5.641}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.923, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1741}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4210.828, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T00:18:12+09:00", "last_error_summary": "[h264 @ 0x55c914327400] error while decoding MB 74 21, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 176, "h264_decode_error_count": 250, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:18:12+09:00", "last_error_summary": "[h264 @ 0x55c914327400] error while decoding MB 74 21, bytestream -5", "last_exit_at": "2026-07-02T00:22:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 242.4, "last_started_at": "2026-07-02T00:18:15+09:00", "pid": 30834, "process": {"exists": false, "pid": 30834}, "restart_count": 201, "restart_count_per_hour": 6, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 202}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 140.43, "sample_count": 2301, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 208963, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:22:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-02T00:22:18+09:00", "last_frame_read_age_sec": 0.125, "last_frame_read_at": "2026-07-02T00:22:17+09:00", "mjpeg": {"age_sec": 0.115, "exists": true, "mtime": "2026-07-02T00:22:17.919091+09:00", "mtime_epoch": 1782919337.919091, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 117140}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:22:18+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 967.582, "name": "celery", "pid": 237, "rss_mb": 964.336, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 230, "pushed_frames": 230, "read_frames": 229, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.689, "pushed": 7.689, "read": 7.656}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.473, "count": 230, "max_ms": 5.363, "p95_ms": 3.956}, "gstreamer_push": {"avg_ms": 1.334, "count": 230, "max_ms": 7.927, "p95_ms": 2.437}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.24, "count": 230, "max_ms": 78.666, "p95_ms": 68.485}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.38, "count": 230, "max_ms": 92.813, "p95_ms": 90.147}, "yolo11_640_fire_11class": {"avg_ms": 60.541, "count": 230, "max_ms": 83.62, "p95_ms": 76.85}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.772, "count": 230, "max_ms": 92.374, "p95_ms": 88.548}}, "truncated_count": 0}, "processing_total": {"avg_ms": 117.031, "count": 230, "max_ms": 157.907, "p95_ms": 141.493}, "read_interval": {"avg_ms": 130.537, "count": 229, "max_ms": 170.776, "p95_ms": 154.425}, "triton_wait": {"avg_ms": 70.569, "count": 230, "max_ms": 92.825, "p95_ms": 90.374}, "visualize": {"avg_ms": 5.053, "count": 230, "max_ms": 6.561, "p95_ms": 5.577}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.913, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1741}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4210.785, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 176, "h264_decode_error_count": 250, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:18:12+09:00", "last_error_summary": "[h264 @ 0x55c914327400] error while decoding MB 74 21, bytestream -5", "last_exit_at": "2026-07-02T00:22:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 242.406, "last_started_at": "2026-07-02T00:18:15+09:00", "pid": 30834, "process": {"exists": false, "pid": 30834}, "restart_count": 201, "restart_count_per_hour": 6, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 202}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 140.43, "sample_count": 2301, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 208963, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:22:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-02T00:22:18+09:00", "last_frame_read_age_sec": 0.13, "last_frame_read_at": "2026-07-02T00:22:17+09:00", "mjpeg": {"age_sec": 0.121, "exists": true, "mtime": "2026-07-02T00:22:17.919091+09:00", "mtime_epoch": 1782919337.919091, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 117140}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:22:18+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 967.582, "name": "celery", "pid": 237, "rss_mb": 964.336, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 230, "pushed_frames": 230, "read_frames": 229, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.688, "pushed": 7.688, "read": 7.654}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.473, "count": 230, "max_ms": 5.363, "p95_ms": 3.956}, "gstreamer_push": {"avg_ms": 1.334, "count": 230, "max_ms": 7.927, "p95_ms": 2.437}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.24, "count": 230, "max_ms": 78.666, "p95_ms": 68.485}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.38, "count": 230, "max_ms": 92.813, "p95_ms": 90.147}, "yolo11_640_fire_11class": {"avg_ms": 60.541, "count": 230, "max_ms": 83.62, "p95_ms": 76.85}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.772, "count": 230, "max_ms": 92.374, "p95_ms": 88.548}}, "truncated_count": 0}, "processing_total": {"avg_ms": 117.031, "count": 230, "max_ms": 157.907, "p95_ms": 141.493}, "read_interval": {"avg_ms": 130.537, "count": 229, "max_ms": 170.776, "p95_ms": 154.425}, "triton_wait": {"avg_ms": 70.569, "count": 230, "max_ms": 92.825, "p95_ms": 90.374}, "visualize": {"avg_ms": 5.053, "count": 230, "max_ms": 6.561, "p95_ms": 5.577}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.918, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1741}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4315.875, "limit_mb": 8796093022207.996}, "detail": {"pid": 30941, "restart_count": 202, "start_count": 203}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 176, "h264_decode_error_count": 250, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:18:12+09:00", "last_error_summary": "[h264 @ 0x55c914327400] error while decoding MB 74 21, bytestream -5", "last_exit_at": "2026-07-02T00:22:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T00:22:20+09:00", "pid": 30941, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.273, "name": "ffmpeg", "pid": 30941, "rss_mb": 139.273, "state": "S (sleeping)", "threads": 106}, "restart_count": 202, "restart_count_per_hour": 7, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 203}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 140.43, "sample_count": 2267, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 208963, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:22:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.542, "last_frame_pushed_at": "2026-07-02T00:22:18+09:00", "last_frame_read_age_sec": 4.659, "last_frame_read_at": "2026-07-02T00:22:17+09:00", "mjpeg": {"age_sec": 4.521, "exists": true, "mtime": "2026-07-02T00:22:18.047093+09:00", "mtime_epoch": 1782919338.0470927, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 116822}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:22:22+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 967.582, "name": "celery", "pid": 237, "rss_mb": 964.336, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 196, "pushed_frames": 196, "read_frames": 195, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.543, "pushed": 6.543, "read": 6.509}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.428, "count": 196, "max_ms": 4.648, "p95_ms": 3.943}, "gstreamer_push": {"avg_ms": 1.353, "count": 196, "max_ms": 7.927, "p95_ms": 2.488}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.311, "count": 196, "max_ms": 78.666, "p95_ms": 68.485}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.581, "count": 196, "max_ms": 92.813, "p95_ms": 90.387}, "yolo11_640_fire_11class": {"avg_ms": 60.542, "count": 196, "max_ms": 83.62, "p95_ms": 76.85}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.975, "count": 196, "max_ms": 92.374, "p95_ms": 88.879}}, "truncated_count": 0}, "processing_total": {"avg_ms": 116.534, "count": 196, "max_ms": 157.907, "p95_ms": 141.493}, "read_interval": {"avg_ms": 130.101, "count": 195, "max_ms": 170.776, "p95_ms": 154.425}, "triton_wait": {"avg_ms": 70.775, "count": 196, "max_ms": 92.825, "p95_ms": 90.643}, "visualize": {"avg_ms": 5.032, "count": 196, "max_ms": 6.561, "p95_ms": 5.642}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.957, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1753}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4210.207, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x555ee923dd00] error while decoding MB 72 26, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 177, "h264_decode_error_count": 251, "h264_decode_error_count_per_hour": 17, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:25:35+09:00", "last_error_summary": "[h264 @ 0x555ee923dd00] error while decoding MB 72 26, bytestream -9", "last_exit_at": "2026-07-02T00:22:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 195.191, "last_started_at": "2026-07-02T00:22:20+09:00", "pid": 30941, "process": {"exists": false, "pid": 30941}, "restart_count": 202, "restart_count_per_hour": 6, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 203}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 141.8, "sample_count": 2223, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 210403, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:25:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.066, "last_frame_pushed_at": "2026-07-02T00:25:35+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-02T00:25:35+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-02T00:25:35.713819+09:00", "mtime_epoch": 1782919535.7138186, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 122685}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:25:35+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 970.293, "name": "celery", "pid": 237, "rss_mb": 965.254, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 222, "pushed_frames": 222, "read_frames": 222, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.417, "pushed": 7.417, "read": 7.417}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.724, "count": 222, "max_ms": 5.41, "p95_ms": 4.159}, "gstreamer_push": {"avg_ms": 1.248, "count": 222, "max_ms": 4.309, "p95_ms": 1.296}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.155, "count": 222, "max_ms": 77.056, "p95_ms": 68.15}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.83, "count": 222, "max_ms": 97.178, "p95_ms": 89.984}, "yolo11_640_fire_11class": {"avg_ms": 61.471, "count": 222, "max_ms": 86.451, "p95_ms": 78.399}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.532, "count": 222, "max_ms": 96.422, "p95_ms": 89.01}}, "truncated_count": 0}, "processing_total": {"avg_ms": 122.231, "count": 222, "max_ms": 155.317, "p95_ms": 141.478}, "read_interval": {"avg_ms": 135.205, "count": 222, "max_ms": 170.856, "p95_ms": 155.365}, "triton_wait": {"avg_ms": 70.982, "count": 222, "max_ms": 97.19, "p95_ms": 89.996}, "visualize": {"avg_ms": 5.228, "count": 222, "max_ms": 6.395, "p95_ms": 5.765}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.93, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1753}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4209.742, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T00:25:35+09:00", "last_error_summary": "[h264 @ 0x555ee923dd00] error while decoding MB 72 26, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 177, "h264_decode_error_count": 251, "h264_decode_error_count_per_hour": 17, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:25:35+09:00", "last_error_summary": "[h264 @ 0x555ee923dd00] error while decoding MB 72 26, bytestream -9", "last_exit_at": "2026-07-02T00:25:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 195.275, "last_started_at": "2026-07-02T00:22:20+09:00", "pid": 30941, "process": {"exists": false, "pid": 30941}, "restart_count": 202, "restart_count_per_hour": 6, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 203}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 141.8, "sample_count": 2223, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 210404, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:25:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T00:25:35+09:00", "last_frame_read_age_sec": 0.139, "last_frame_read_at": "2026-07-02T00:25:35+09:00", "mjpeg": {"age_sec": 0.128, "exists": true, "mtime": "2026-07-02T00:25:35.713819+09:00", "mtime_epoch": 1782919535.7138186, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 122685}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:25:35+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 970.293, "name": "celery", "pid": 237, "rss_mb": 965.258, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 222, "pushed_frames": 222, "read_frames": 221, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.428, "pushed": 7.428, "read": 7.395}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.722, "count": 222, "max_ms": 5.41, "p95_ms": 4.159}, "gstreamer_push": {"avg_ms": 1.247, "count": 222, "max_ms": 4.309, "p95_ms": 1.296}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.238, "count": 222, "max_ms": 77.056, "p95_ms": 68.784}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.786, "count": 222, "max_ms": 97.178, "p95_ms": 89.91}, "yolo11_640_fire_11class": {"avg_ms": 61.442, "count": 222, "max_ms": 86.451, "p95_ms": 78.399}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.5, "count": 222, "max_ms": 96.422, "p95_ms": 89.01}}, "truncated_count": 0}, "processing_total": {"avg_ms": 122.186, "count": 222, "max_ms": 155.317, "p95_ms": 141.478}, "read_interval": {"avg_ms": 135.121, "count": 221, "max_ms": 170.856, "p95_ms": 155.365}, "triton_wait": {"avg_ms": 70.944, "count": 222, "max_ms": 97.19, "p95_ms": 89.954}, "visualize": {"avg_ms": 5.228, "count": 222, "max_ms": 6.395, "p95_ms": 5.765}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.887, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1753}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4209.734, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 177, "h264_decode_error_count": 251, "h264_decode_error_count_per_hour": 17, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:25:35+09:00", "last_error_summary": "[h264 @ 0x555ee923dd00] error while decoding MB 72 26, bytestream -9", "last_exit_at": "2026-07-02T00:25:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 195.28, "last_started_at": "2026-07-02T00:22:20+09:00", "pid": 30941, "process": {"exists": false, "pid": 30941}, "restart_count": 202, "restart_count_per_hour": 6, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 203}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 141.8, "sample_count": 2223, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 210404, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:25:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-02T00:25:35+09:00", "last_frame_read_age_sec": 0.145, "last_frame_read_at": "2026-07-02T00:25:35+09:00", "mjpeg": {"age_sec": 0.134, "exists": true, "mtime": "2026-07-02T00:25:35.713819+09:00", "mtime_epoch": 1782919535.7138186, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 122685}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:25:35+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 970.293, "name": "celery", "pid": 237, "rss_mb": 965.258, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 222, "pushed_frames": 222, "read_frames": 221, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.427, "pushed": 7.427, "read": 7.393}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.722, "count": 222, "max_ms": 5.41, "p95_ms": 4.159}, "gstreamer_push": {"avg_ms": 1.247, "count": 222, "max_ms": 4.309, "p95_ms": 1.296}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.238, "count": 222, "max_ms": 77.056, "p95_ms": 68.784}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.786, "count": 222, "max_ms": 97.178, "p95_ms": 89.91}, "yolo11_640_fire_11class": {"avg_ms": 61.442, "count": 222, "max_ms": 86.451, "p95_ms": 78.399}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.5, "count": 222, "max_ms": 96.422, "p95_ms": 89.01}}, "truncated_count": 0}, "processing_total": {"avg_ms": 122.186, "count": 222, "max_ms": 155.317, "p95_ms": 141.478}, "read_interval": {"avg_ms": 135.121, "count": 221, "max_ms": 170.856, "p95_ms": 155.365}, "triton_wait": {"avg_ms": 70.944, "count": 222, "max_ms": 97.19, "p95_ms": 89.954}, "visualize": {"avg_ms": 5.228, "count": 222, "max_ms": 6.395, "p95_ms": 5.765}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.892, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1753}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4315.523, "limit_mb": 8796093022207.996}, "detail": {"pid": 31048, "restart_count": 203, "start_count": 204}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 177, "h264_decode_error_count": 251, "h264_decode_error_count_per_hour": 17, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:25:35+09:00", "last_error_summary": "[h264 @ 0x555ee923dd00] error while decoding MB 72 26, bytestream -9", "last_exit_at": "2026-07-02T00:25:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T00:25:37+09:00", "pid": 31048, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.512, "name": "ffmpeg", "pid": 31048, "rss_mb": 138.512, "state": "S (sleeping)", "threads": 106}, "restart_count": 203, "restart_count_per_hour": 7, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 204}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 141.799, "sample_count": 2189, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 210404, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:25:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.159, "last_frame_pushed_at": "2026-07-02T00:25:35+09:00", "last_frame_read_age_sec": 4.292, "last_frame_read_at": "2026-07-02T00:25:35+09:00", "mjpeg": {"age_sec": 4.142, "exists": true, "mtime": "2026-07-02T00:25:35.853821+09:00", "mtime_epoch": 1782919535.8538206, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 121616}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:25:40+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 970.293, "name": "celery", "pid": 237, "rss_mb": 965.258, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 191, "pushed_frames": 191, "read_frames": 190, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.384, "pushed": 6.384, "read": 6.351}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.723, "count": 191, "max_ms": 5.41, "p95_ms": 4.168}, "gstreamer_push": {"avg_ms": 1.257, "count": 191, "max_ms": 4.309, "p95_ms": 1.296}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.211, "count": 191, "max_ms": 77.056, "p95_ms": 68.784}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.049, "count": 191, "max_ms": 97.178, "p95_ms": 89.91}, "yolo11_640_fire_11class": {"avg_ms": 61.692, "count": 191, "max_ms": 86.451, "p95_ms": 78.268}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.823, "count": 191, "max_ms": 96.422, "p95_ms": 88.935}}, "truncated_count": 0}, "processing_total": {"avg_ms": 122.542, "count": 191, "max_ms": 155.317, "p95_ms": 141.488}, "read_interval": {"avg_ms": 135.487, "count": 190, "max_ms": 170.856, "p95_ms": 156.371}, "triton_wait": {"avg_ms": 71.218, "count": 191, "max_ms": 97.19, "p95_ms": 89.954}, "visualize": {"avg_ms": 5.23, "count": 191, "max_ms": 6.395, "p95_ms": 5.772}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.916, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1765}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4207.238, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x563002060e40] error while decoding MB 5 10, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 178, "h264_decode_error_count": 252, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:28:52+09:00", "last_error_summary": "[h264 @ 0x563002060e40] error while decoding MB 5 10, bytestream -9", "last_exit_at": "2026-07-02T00:25:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 194.806, "last_started_at": "2026-07-02T00:25:37+09:00", "pid": 31048, "process": {"exists": false, "pid": 31048}, "restart_count": 203, "restart_count_per_hour": 7, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 204}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 140.902, "sample_count": 2195, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 211826, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:28:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.067, "last_frame_pushed_at": "2026-07-02T00:28:52+09:00", "last_frame_read_age_sec": 0.056, "last_frame_read_at": "2026-07-02T00:28:52+09:00", "mjpeg": {"age_sec": 0.052, "exists": true, "mtime": "2026-07-02T00:28:52.748636+09:00", "mtime_epoch": 1782919732.7486362, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 117116}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:28:52+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 970.77, "name": "celery", "pid": 237, "rss_mb": 961.07, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 213, "pushed_frames": 213, "read_frames": 213, "select_timeouts": 1, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.115, "pushed": 7.115, "read": 7.115}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.698, "count": 213, "max_ms": 5.361, "p95_ms": 4.029}, "gstreamer_push": {"avg_ms": 1.245, "count": 213, "max_ms": 4.458, "p95_ms": 1.343}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.138, "count": 213, "max_ms": 76.518, "p95_ms": 71.343}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.177, "count": 213, "max_ms": 92.006, "p95_ms": 88.318}, "yolo11_640_fire_11class": {"avg_ms": 60.393, "count": 213, "max_ms": 81.012, "p95_ms": 75.241}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.863, "count": 213, "max_ms": 93.551, "p95_ms": 88.147}}, "truncated_count": 0}, "processing_total": {"avg_ms": 120.364, "count": 213, "max_ms": 162.43, "p95_ms": 141.104}, "read_interval": {"avg_ms": 140.996, "count": 213, "max_ms": 1772.252, "p95_ms": 155.609}, "triton_wait": {"avg_ms": 69.826, "count": 213, "max_ms": 93.564, "p95_ms": 89.056}, "visualize": {"avg_ms": 5.174, "count": 213, "max_ms": 6.758, "p95_ms": 5.391}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.938, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1765}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4203.234, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T00:28:52+09:00", "last_error_summary": "[h264 @ 0x563002060e40] error while decoding MB 5 10, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 178, "h264_decode_error_count": 252, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:28:52+09:00", "last_error_summary": "[h264 @ 0x563002060e40] error while decoding MB 5 10, bytestream -9", "last_exit_at": "2026-07-02T00:28:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 194.905, "last_started_at": "2026-07-02T00:25:37+09:00", "pid": 31048, "process": {"exists": false, "pid": 31048}, "restart_count": 203, "restart_count_per_hour": 7, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 204}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 140.914, "sample_count": 2195, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 211827, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:28:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T00:28:52+09:00", "last_frame_read_age_sec": 0.155, "last_frame_read_at": "2026-07-02T00:28:52+09:00", "mjpeg": {"age_sec": 0.151, "exists": true, "mtime": "2026-07-02T00:28:52.748636+09:00", "mtime_epoch": 1782919732.7486362, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 117116}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:28:52+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 970.77, "name": "celery", "pid": 237, "rss_mb": 956.797, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 213, "pushed_frames": 213, "read_frames": 212, "select_timeouts": 1, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.119, "pushed": 7.119, "read": 7.086}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.698, "count": 213, "max_ms": 5.361, "p95_ms": 4.029}, "gstreamer_push": {"avg_ms": 1.238, "count": 213, "max_ms": 4.458, "p95_ms": 1.315}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.108, "count": 213, "max_ms": 76.518, "p95_ms": 71.26}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.189, "count": 213, "max_ms": 92.006, "p95_ms": 88.318}, "yolo11_640_fire_11class": {"avg_ms": 60.416, "count": 213, "max_ms": 81.012, "p95_ms": 75.277}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.875, "count": 213, "max_ms": 93.551, "p95_ms": 88.398}}, "truncated_count": 0}, "processing_total": {"avg_ms": 120.358, "count": 213, "max_ms": 162.43, "p95_ms": 141.104}, "read_interval": {"avg_ms": 140.889, "count": 212, "max_ms": 1772.252, "p95_ms": 155.305}, "triton_wait": {"avg_ms": 69.838, "count": 213, "max_ms": 93.564, "p95_ms": 89.056}, "visualize": {"avg_ms": 5.174, "count": 213, "max_ms": 6.758, "p95_ms": 5.391}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.92, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1765}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4203.227, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 178, "h264_decode_error_count": 252, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:28:52+09:00", "last_error_summary": "[h264 @ 0x563002060e40] error while decoding MB 5 10, bytestream -9", "last_exit_at": "2026-07-02T00:28:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 194.91, "last_started_at": "2026-07-02T00:25:37+09:00", "pid": 31048, "process": {"exists": false, "pid": 31048}, "restart_count": 203, "restart_count_per_hour": 7, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 204}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 140.914, "sample_count": 2195, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 211827, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:28:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-02T00:28:52+09:00", "last_frame_read_age_sec": 0.16, "last_frame_read_at": "2026-07-02T00:28:52+09:00", "mjpeg": {"age_sec": 0.156, "exists": true, "mtime": "2026-07-02T00:28:52.748636+09:00", "mtime_epoch": 1782919732.7486362, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 117116}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:28:52+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 970.77, "name": "celery", "pid": 237, "rss_mb": 956.797, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 213, "pushed_frames": 213, "read_frames": 212, "select_timeouts": 1, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.118, "pushed": 7.118, "read": 7.084}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.698, "count": 213, "max_ms": 5.361, "p95_ms": 4.029}, "gstreamer_push": {"avg_ms": 1.238, "count": 213, "max_ms": 4.458, "p95_ms": 1.315}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.108, "count": 213, "max_ms": 76.518, "p95_ms": 71.26}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.189, "count": 213, "max_ms": 92.006, "p95_ms": 88.318}, "yolo11_640_fire_11class": {"avg_ms": 60.416, "count": 213, "max_ms": 81.012, "p95_ms": 75.277}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.875, "count": 213, "max_ms": 93.551, "p95_ms": 88.398}}, "truncated_count": 0}, "processing_total": {"avg_ms": 120.358, "count": 213, "max_ms": 162.43, "p95_ms": 141.104}, "read_interval": {"avg_ms": 140.889, "count": 212, "max_ms": 1772.252, "p95_ms": 155.305}, "triton_wait": {"avg_ms": 69.838, "count": 213, "max_ms": 93.564, "p95_ms": 89.056}, "visualize": {"avg_ms": 5.174, "count": 213, "max_ms": 6.758, "p95_ms": 5.391}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.925, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1765}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4312.965, "limit_mb": 8796093022207.996}, "detail": {"pid": 31155, "restart_count": 204, "start_count": 205}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 178, "h264_decode_error_count": 252, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:28:52+09:00", "last_error_summary": "[h264 @ 0x563002060e40] error while decoding MB 5 10, bytestream -9", "last_exit_at": "2026-07-02T00:28:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T00:28:55+09:00", "pid": 31155, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.652, "name": "ffmpeg", "pid": 31155, "rss_mb": 141.652, "state": "S (sleeping)", "threads": 106}, "restart_count": 204, "restart_count_per_hour": 8, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 205}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 140.899, "sample_count": 2164, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 211827, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:28:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.411, "last_frame_pushed_at": "2026-07-02T00:28:52+09:00", "last_frame_read_age_sec": 4.562, "last_frame_read_at": "2026-07-02T00:28:52+09:00", "mjpeg": {"age_sec": 4.398, "exists": true, "mtime": "2026-07-02T00:28:52.908638+09:00", "mtime_epoch": 1782919732.9086385, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 115429}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:28:57+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 970.77, "name": "celery", "pid": 237, "rss_mb": 956.797, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 180, "pushed_frames": 180, "read_frames": 179, "select_timeouts": 1, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.019, "pushed": 6.019, "read": 5.986}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.687, "count": 180, "max_ms": 5.361, "p95_ms": 4.024}, "gstreamer_push": {"avg_ms": 1.248, "count": 180, "max_ms": 4.458, "p95_ms": 1.343}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.018, "count": 180, "max_ms": 76.518, "p95_ms": 69.852}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.402, "count": 180, "max_ms": 92.006, "p95_ms": 88.318}, "yolo11_640_fire_11class": {"avg_ms": 60.551, "count": 180, "max_ms": 81.012, "p95_ms": 75.081}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.988, "count": 180, "max_ms": 90.716, "p95_ms": 88.042}}, "truncated_count": 0}, "processing_total": {"avg_ms": 120.309, "count": 180, "max_ms": 162.43, "p95_ms": 141.094}, "read_interval": {"avg_ms": 142.219, "count": 179, "max_ms": 1772.252, "p95_ms": 154.992}, "triton_wait": {"avg_ms": 69.992, "count": 180, "max_ms": 92.019, "p95_ms": 88.678}, "visualize": {"avg_ms": 5.152, "count": 180, "max_ms": 6.327, "p95_ms": 5.332}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.903, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1783}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4207.281, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55d9641f9fc0] error while decoding MB 59 15, bytestream -12", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 179, "h264_decode_error_count": 253, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:33:25+09:00", "last_error_summary": "[h264 @ 0x55d9641f9fc0] error while decoding MB 59 15, bytestream -12", "last_exit_at": "2026-07-02T00:28:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 269.916, "last_started_at": "2026-07-02T00:28:55+09:00", "pid": 31155, "process": {"exists": false, "pid": 31155}, "restart_count": 204, "restart_count_per_hour": 8, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 205}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 139.936, "sample_count": 2378, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 214009, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:33:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-02T00:33:25+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-02T00:33:25+09:00", "mjpeg": {"age_sec": 0.049, "exists": true, "mtime": "2026-07-02T00:33:25.172640+09:00", "mtime_epoch": 1782920005.1726396, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 112629}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:33:25+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 970.77, "name": "celery", "pid": 237, "rss_mb": 964.277, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 269, "pushed_frames": 269, "read_frames": 269, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.971, "pushed": 8.971, "read": 8.971}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 2.514, "count": 268, "max_ms": 4.782, "p95_ms": 4.125}, "gstreamer_push": {"avg_ms": 1.463, "count": 269, "max_ms": 5.943, "p95_ms": 3.393}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.711, "count": 268, "max_ms": 76.846, "p95_ms": 71.57}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.617, "count": 268, "max_ms": 91.396, "p95_ms": 86.264}, "yolo11_640_fire_11class": {"avg_ms": 62.084, "count": 268, "max_ms": 79.597, "p95_ms": 75.215}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.969, "count": 268, "max_ms": 91.475, "p95_ms": 87.986}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.372, "count": 269, "max_ms": 150.951, "p95_ms": 138.182}, "read_interval": {"avg_ms": 111.69, "count": 269, "max_ms": 163.179, "p95_ms": 151.276}, "triton_wait": {"avg_ms": 70.863, "count": 268, "max_ms": 91.487, "p95_ms": 88.562}, "visualize": {"avg_ms": 4.218, "count": 268, "max_ms": 5.977, "p95_ms": 5.231}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.985, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1783}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4208.516, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T00:33:25+09:00", "last_error_summary": "[h264 @ 0x55d9641f9fc0] error while decoding MB 59 15, bytestream -12", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 179, "h264_decode_error_count": 253, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:33:25+09:00", "last_error_summary": "[h264 @ 0x55d9641f9fc0] error while decoding MB 59 15, bytestream -12", "last_exit_at": "2026-07-02T00:33:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 269.985, "last_started_at": "2026-07-02T00:28:55+09:00", "pid": 31155, "process": {"exists": false, "pid": 31155}, "restart_count": 204, "restart_count_per_hour": 8, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 205}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 139.936, "sample_count": 2378, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 214010, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:33:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T00:33:25+09:00", "last_frame_read_age_sec": 0.122, "last_frame_read_at": "2026-07-02T00:33:25+09:00", "mjpeg": {"age_sec": 0.118, "exists": true, "mtime": "2026-07-02T00:33:25.172640+09:00", "mtime_epoch": 1782920005.1726396, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 112629}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:33:25+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 970.77, "name": "celery", "pid": 237, "rss_mb": 964.277, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 269, "pushed_frames": 269, "read_frames": 268, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.994, "pushed": 8.994, "read": 8.961}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 2.532, "count": 269, "max_ms": 7.368, "p95_ms": 4.125}, "gstreamer_push": {"avg_ms": 1.462, "count": 269, "max_ms": 5.943, "p95_ms": 3.393}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.69, "count": 269, "max_ms": 76.846, "p95_ms": 71.57}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.582, "count": 269, "max_ms": 91.396, "p95_ms": 86.264}, "yolo11_640_fire_11class": {"avg_ms": 62.069, "count": 269, "max_ms": 79.597, "p95_ms": 75.215}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.925, "count": 269, "max_ms": 91.475, "p95_ms": 87.986}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.379, "count": 269, "max_ms": 150.951, "p95_ms": 138.182}, "read_interval": {"avg_ms": 111.637, "count": 268, "max_ms": 163.179, "p95_ms": 151.276}, "triton_wait": {"avg_ms": 70.82, "count": 269, "max_ms": 91.487, "p95_ms": 88.562}, "visualize": {"avg_ms": 4.226, "count": 269, "max_ms": 6.411, "p95_ms": 5.235}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.908, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1783}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4208.883, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 179, "h264_decode_error_count": 253, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:33:25+09:00", "last_error_summary": "[h264 @ 0x55d9641f9fc0] error while decoding MB 59 15, bytestream -12", "last_exit_at": "2026-07-02T00:33:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 269.991, "last_started_at": "2026-07-02T00:28:55+09:00", "pid": 31155, "process": {"exists": false, "pid": 31155}, "restart_count": 204, "restart_count_per_hour": 8, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 205}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 139.936, "sample_count": 2378, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 214010, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:33:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-02T00:33:25+09:00", "last_frame_read_age_sec": 0.128, "last_frame_read_at": "2026-07-02T00:33:25+09:00", "mjpeg": {"age_sec": 0.124, "exists": true, "mtime": "2026-07-02T00:33:25.172640+09:00", "mtime_epoch": 1782920005.1726396, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 112629}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:33:25+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 970.77, "name": "celery", "pid": 237, "rss_mb": 964.277, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 269, "pushed_frames": 269, "read_frames": 268, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.992, "pushed": 8.992, "read": 8.959}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 2.532, "count": 269, "max_ms": 7.368, "p95_ms": 4.125}, "gstreamer_push": {"avg_ms": 1.462, "count": 269, "max_ms": 5.943, "p95_ms": 3.393}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.69, "count": 269, "max_ms": 76.846, "p95_ms": 71.57}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.582, "count": 269, "max_ms": 91.396, "p95_ms": 86.264}, "yolo11_640_fire_11class": {"avg_ms": 62.069, "count": 269, "max_ms": 79.597, "p95_ms": 75.215}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.925, "count": 269, "max_ms": 91.475, "p95_ms": 87.986}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.379, "count": 269, "max_ms": 150.951, "p95_ms": 138.182}, "read_interval": {"avg_ms": 111.637, "count": 268, "max_ms": 163.179, "p95_ms": 151.276}, "triton_wait": {"avg_ms": 70.82, "count": 269, "max_ms": 91.487, "p95_ms": 88.562}, "visualize": {"avg_ms": 4.226, "count": 269, "max_ms": 6.411, "p95_ms": 5.235}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.914, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1783}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4316.785, "limit_mb": 8796093022207.996}, "detail": {"pid": 31262, "restart_count": 205, "start_count": 206}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 179, "h264_decode_error_count": 253, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:33:25+09:00", "last_error_summary": "[h264 @ 0x55d9641f9fc0] error while decoding MB 59 15, bytestream -12", "last_exit_at": "2026-07-02T00:33:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T00:33:27+09:00", "pid": 31262, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.055, "name": "ffmpeg", "pid": 31262, "rss_mb": 139.055, "state": "S (sleeping)", "threads": 106}, "restart_count": 205, "restart_count_per_hour": 9, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 206}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 139.874, "sample_count": 2347, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 214010, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:33:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.186, "last_frame_pushed_at": "2026-07-02T00:33:25+09:00", "last_frame_read_age_sec": 4.303, "last_frame_read_at": "2026-07-02T00:33:25+09:00", "mjpeg": {"age_sec": 4.171, "exists": true, "mtime": "2026-07-02T00:33:25.300642+09:00", "mtime_epoch": 1782920005.3006415, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 114295}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:33:29+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 970.77, "name": "celery", "pid": 237, "rss_mb": 964.277, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 227, "pushed_frames": 227, "read_frames": 226, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.575, "pushed": 7.575, "read": 7.542}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 2.619, "count": 227, "max_ms": 7.368, "p95_ms": 4.127}, "gstreamer_push": {"avg_ms": 1.442, "count": 227, "max_ms": 5.943, "p95_ms": 2.66}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.085, "count": 227, "max_ms": 76.846, "p95_ms": 72.157}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.267, "count": 227, "max_ms": 91.396, "p95_ms": 86.408}, "yolo11_640_fire_11class": {"avg_ms": 62.767, "count": 227, "max_ms": 79.597, "p95_ms": 75.314}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.91, "count": 227, "max_ms": 91.475, "p95_ms": 88.453}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.741, "count": 227, "max_ms": 150.951, "p95_ms": 138.289}, "read_interval": {"avg_ms": 113.913, "count": 226, "max_ms": 163.179, "p95_ms": 151.375}, "triton_wait": {"avg_ms": 71.78, "count": 227, "max_ms": 91.487, "p95_ms": 88.597}, "visualize": {"avg_ms": 4.28, "count": 227, "max_ms": 6.411, "p95_ms": 5.235}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.965, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1799}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4214.195, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 6, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x557fb9257380] cabac decode of qscale diff failed at 17 21 [h264 @ 0x557fb9257380] error while decoding MB 17 21, bytestream 85247 [h264 @ 0x557fb9fe3440] top block unavailable for requested intra mode [h264 @ 0x557fb9fe3440] error while decoding MB 35 0, bytestream 78859 [h264 @ 0x557fb9372700] cabac decode of qscale diff failed at 67 17 [h264 @ 0x557fb9372700] error while decoding MB 67 17, bytestream 5639 [h264 @ 0x557fb934a440] error while decoding MB 13 43, bytestream -6 [in#0/rtsp", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 180, "h264_decode_error_count": 259, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:37:13+09:00", "last_error_summary": "[h264 @ 0x557fb9257380] cabac decode of qscale diff failed at 17 21 [h264 @ 0x557fb9257380] error while decoding MB 17 21, bytestream 85247 [h264 @ 0x557fb9fe3440] top block unavailable for requested intra mode [h264 @ 0x557fb9fe3440] error while decoding MB 35 0, bytestream 78859 [h264 @ 0x557fb9372700] cabac decode of qscale diff failed at 67 17 [h264 @ 0x557fb9372700] error while decoding MB 67 17, bytestream 5639 [h264 @ 0x557fb934a440] error while decoding MB 13 43, bytestream -6 [in#0/rtsp", "last_exit_at": "2026-07-02T00:33:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 226.526, "last_started_at": "2026-07-02T00:33:27+09:00", "pid": 31262, "process": {"exists": false, "pid": 31262}, "restart_count": 205, "restart_count_per_hour": 8, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 206}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 136.264, "sample_count": 2539, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 215930, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:37:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.069, "last_frame_pushed_at": "2026-07-02T00:37:13+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-02T00:37:13+09:00", "mjpeg": {"age_sec": 0.052, "exists": true, "mtime": "2026-07-02T00:37:13.943843+09:00", "mtime_epoch": 1782920233.9438434, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 106531}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:37:14+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 970.77, "name": "celery", "pid": 237, "rss_mb": 961.672, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 102, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.403, "pushed": 3.403, "read": 3.403}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.985, "count": 102, "max_ms": 4.413, "p95_ms": 4.087}, "gstreamer_push": {"avg_ms": 1.46, "count": 102, "max_ms": 4.428, "p95_ms": 1.66}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.705, "count": 102, "max_ms": 78.81, "p95_ms": 75.057}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.746, "count": 102, "max_ms": 85.716, "p95_ms": 82.549}, "yolo11_640_fire_11class": {"avg_ms": 65.773, "count": 102, "max_ms": 77.942, "p95_ms": 76.538}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.547, "count": 102, "max_ms": 89.853, "p95_ms": 87.454}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.952, "count": 102, "max_ms": 138.103, "p95_ms": 128.76}, "read_interval": {"avg_ms": 293.929, "count": 102, "max_ms": 19148.303, "p95_ms": 148.332}, "triton_wait": {"avg_ms": 73.855, "count": 102, "max_ms": 89.866, "p95_ms": 87.465}, "visualize": {"avg_ms": 3.99, "count": 102, "max_ms": 5.533, "p95_ms": 5.218}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.975, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1799}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4214.082, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T00:37:13+09:00", "last_error_summary": "[h264 @ 0x557fb9257380] cabac decode of qscale diff failed at 17 21 [h264 @ 0x557fb9257380] error while decoding MB 17 21, bytestream 85247 [h264 @ 0x557fb9fe3440] top block unavailable for requested intra mode [h264 @ 0x557fb9fe3440] error while decoding MB 35 0, bytestream 78859 [h264 @ 0x557fb9372700] cabac decode of qscale diff failed at 67 17 [h264 @ 0x557fb9372700] error while decoding MB 67 17, bytestream 5639 [h264 @ 0x557fb934a440] error while decoding MB 13 43, bytestream -6 [in#0/rtsp", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 180, "h264_decode_error_count": 259, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:37:13+09:00", "last_error_summary": "[h264 @ 0x557fb9257380] cabac decode of qscale diff failed at 17 21 [h264 @ 0x557fb9257380] error while decoding MB 17 21, bytestream 85247 [h264 @ 0x557fb9fe3440] top block unavailable for requested intra mode [h264 @ 0x557fb9fe3440] error while decoding MB 35 0, bytestream 78859 [h264 @ 0x557fb9372700] cabac decode of qscale diff failed at 67 17 [h264 @ 0x557fb9372700] error while decoding MB 67 17, bytestream 5639 [h264 @ 0x557fb934a440] error while decoding MB 13 43, bytestream -6 [in#0/rtsp", "last_exit_at": "2026-07-02T00:37:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 226.545, "last_started_at": "2026-07-02T00:33:27+09:00", "pid": 31262, "process": {"exists": false, "pid": 31262}, "restart_count": 205, "restart_count_per_hour": 8, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 206}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 136.264, "sample_count": 2540, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 215931, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:37:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T00:37:14+09:00", "last_frame_read_age_sec": 0.075, "last_frame_read_at": "2026-07-02T00:37:13+09:00", "mjpeg": {"age_sec": 0.072, "exists": true, "mtime": "2026-07-02T00:37:13.943843+09:00", "mtime_epoch": 1782920233.9438434, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 106531}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:37:14+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 970.77, "name": "celery", "pid": 237, "rss_mb": 961.672, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.434, "pushed": 3.434, "read": 3.401}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.981, "count": 103, "max_ms": 4.413, "p95_ms": 4.087}, "gstreamer_push": {"avg_ms": 1.459, "count": 103, "max_ms": 4.428, "p95_ms": 1.66}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.572, "count": 103, "max_ms": 78.81, "p95_ms": 75.057}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.561, "count": 103, "max_ms": 85.716, "p95_ms": 82.549}, "yolo11_640_fire_11class": {"avg_ms": 65.615, "count": 103, "max_ms": 77.942, "p95_ms": 76.538}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.32, "count": 103, "max_ms": 89.853, "p95_ms": 87.454}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.708, "count": 103, "max_ms": 138.103, "p95_ms": 128.76}, "read_interval": {"avg_ms": 293.929, "count": 102, "max_ms": 19148.303, "p95_ms": 148.332}, "triton_wait": {"avg_ms": 73.631, "count": 103, "max_ms": 89.866, "p95_ms": 87.465}, "visualize": {"avg_ms": 4.01, "count": 103, "max_ms": 6.084, "p95_ms": 5.232}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1799}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4213.863, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 180, "h264_decode_error_count": 259, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:37:13+09:00", "last_error_summary": "[h264 @ 0x557fb9257380] cabac decode of qscale diff failed at 17 21 [h264 @ 0x557fb9257380] error while decoding MB 17 21, bytestream 85247 [h264 @ 0x557fb9fe3440] top block unavailable for requested intra mode [h264 @ 0x557fb9fe3440] error while decoding MB 35 0, bytestream 78859 [h264 @ 0x557fb9372700] cabac decode of qscale diff failed at 67 17 [h264 @ 0x557fb9372700] error while decoding MB 67 17, bytestream 5639 [h264 @ 0x557fb934a440] error while decoding MB 13 43, bytestream -6 [in#0/rtsp", "last_exit_at": "2026-07-02T00:37:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 226.554, "last_started_at": "2026-07-02T00:33:27+09:00", "pid": 31262, "process": {"exists": false, "pid": 31262}, "restart_count": 205, "restart_count_per_hour": 8, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 206}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 136.264, "sample_count": 2540, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 215931, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:37:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-02T00:37:14+09:00", "last_frame_read_age_sec": 0.084, "last_frame_read_at": "2026-07-02T00:37:13+09:00", "mjpeg": {"age_sec": 0.081, "exists": true, "mtime": "2026-07-02T00:37:13.943843+09:00", "mtime_epoch": 1782920233.9438434, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 106531}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:37:14+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 970.77, "name": "celery", "pid": 237, "rss_mb": 961.672, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 102, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.401, "pushed": 3.401, "read": 3.401}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.986, "count": 102, "max_ms": 4.413, "p95_ms": 4.087}, "gstreamer_push": {"avg_ms": 1.459, "count": 102, "max_ms": 4.428, "p95_ms": 1.66}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.701, "count": 102, "max_ms": 78.81, "p95_ms": 75.057}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.748, "count": 102, "max_ms": 85.716, "p95_ms": 82.549}, "yolo11_640_fire_11class": {"avg_ms": 65.769, "count": 102, "max_ms": 77.942, "p95_ms": 76.538}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.549, "count": 102, "max_ms": 89.853, "p95_ms": 87.454}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.998, "count": 102, "max_ms": 138.103, "p95_ms": 128.76}, "read_interval": {"avg_ms": 293.929, "count": 102, "max_ms": 19148.303, "p95_ms": 148.332}, "triton_wait": {"avg_ms": 73.857, "count": 102, "max_ms": 89.866, "p95_ms": 87.465}, "visualize": {"avg_ms": 4.013, "count": 102, "max_ms": 6.084, "p95_ms": 5.232}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.987, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1799}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4321.57, "limit_mb": 8796093022207.996}, "detail": {"pid": 31369, "restart_count": 206, "start_count": 207}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 180, "h264_decode_error_count": 259, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:37:13+09:00", "last_error_summary": "[h264 @ 0x557fb9257380] cabac decode of qscale diff failed at 17 21 [h264 @ 0x557fb9257380] error while decoding MB 17 21, bytestream 85247 [h264 @ 0x557fb9fe3440] top block unavailable for requested intra mode [h264 @ 0x557fb9fe3440] error while decoding MB 35 0, bytestream 78859 [h264 @ 0x557fb9372700] cabac decode of qscale diff failed at 67 17 [h264 @ 0x557fb9372700] error while decoding MB 67 17, bytestream 5639 [h264 @ 0x557fb934a440] error while decoding MB 13 43, bytestream -6 [in#0/rtsp", "last_exit_at": "2026-07-02T00:37:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T00:37:16+09:00", "pid": 31369, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.477, "name": "ffmpeg", "pid": 31369, "rss_mb": 141.477, "state": "S (sleeping)", "threads": 106}, "restart_count": 206, "restart_count_per_hour": 9, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 207}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 136.249, "sample_count": 2504, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 215931, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:37:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.351, "last_frame_pushed_at": "2026-07-02T00:37:14+09:00", "last_frame_read_age_sec": 4.419, "last_frame_read_at": "2026-07-02T00:37:13+09:00", "mjpeg": {"age_sec": 4.332, "exists": true, "mtime": "2026-07-02T00:37:14.027845+09:00", "mtime_epoch": 1782920234.0278447, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 105189}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:37:18+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 970.77, "name": "celery", "pid": 237, "rss_mb": 961.672, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 62, "pushed_frames": 62, "read_frames": 61, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.068, "pushed": 2.068, "read": 2.035}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.871, "count": 62, "max_ms": 4.413, "p95_ms": 4.039}, "gstreamer_push": {"avg_ms": 1.532, "count": 62, "max_ms": 4.428, "p95_ms": 1.66}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.677, "count": 62, "max_ms": 78.81, "p95_ms": 75.889}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.053, "count": 62, "max_ms": 85.716, "p95_ms": 81.941}, "yolo11_640_fire_11class": {"avg_ms": 67.652, "count": 62, "max_ms": 77.942, "p95_ms": 77.233}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.184, "count": 62, "max_ms": 89.853, "p95_ms": 88.66}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.193, "count": 62, "max_ms": 138.103, "p95_ms": 114.872}, "read_interval": {"avg_ms": 420.121, "count": 61, "max_ms": 19148.303, "p95_ms": 131.171}, "triton_wait": {"avg_ms": 76.386, "count": 62, "max_ms": 89.866, "p95_ms": 88.673}, "visualize": {"avg_ms": 3.964, "count": 62, "max_ms": 6.084, "p95_ms": 5.185}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.981, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1835}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4219.367, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 8, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5617f61a36c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5617f61a36c0] error while decoding MB 0 9, bytestream 6253 [h264 @ 0x5617f627c240] error while decoding MB 13 10, bytestream -13 [h264 @ 0x5617f61a36c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5617f61a36c0] error while decoding MB 0 5, bytestream 912 [h264 @ 0x5617f6254040] reference picture missing during reorder [h264 @ 0x5617f6254040] Missing reference picture, default is 65557 [h26", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 181, "h264_decode_error_count": 267, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:45:17+09:00", "last_error_summary": "[h264 @ 0x5617f61a36c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5617f61a36c0] error while decoding MB 0 9, bytestream 6253 [h264 @ 0x5617f627c240] error while decoding MB 13 10, bytestream -13 [h264 @ 0x5617f61a36c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5617f61a36c0] error while decoding MB 0 5, bytestream 912 [h264 @ 0x5617f6254040] reference picture missing during reorder [h264 @ 0x5617f6254040] Missing reference picture, default is 65557 [h26", "last_exit_at": "2026-07-02T00:37:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 481.092, "last_started_at": "2026-07-02T00:37:16+09:00", "pid": 31369, "process": {"exists": false, "pid": 31369}, "restart_count": 206, "restart_count_per_hour": 9, "rtsp_timeout_count": 50, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 207}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 136.58, "sample_count": 2579, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 220243, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:45:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.078, "last_frame_pushed_at": "2026-07-02T00:45:17+09:00", "last_frame_read_age_sec": 0.063, "last_frame_read_at": "2026-07-02T00:45:17+09:00", "mjpeg": {"age_sec": 0.06, "exists": true, "mtime": "2026-07-02T00:45:17.390469+09:00", "mtime_epoch": 1782920717.390469, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 127937}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:45:17+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 972.41, "name": "celery", "pid": 237, "rss_mb": 969.215, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 86, "pushed_frames": 86, "read_frames": 86, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.874, "pushed": 2.874, "read": 2.874}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.099, "count": 86, "max_ms": 4.252, "p95_ms": 4.005}, "gstreamer_push": {"avg_ms": 1.578, "count": 86, "max_ms": 6.47, "p95_ms": 3.935}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.091, "count": 86, "max_ms": 77.735, "p95_ms": 69.763}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.655, "count": 86, "max_ms": 88.697, "p95_ms": 83.101}, "yolo11_640_fire_11class": {"avg_ms": 67.348, "count": 86, "max_ms": 78.597, "p95_ms": 76.992}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.952, "count": 86, "max_ms": 89.445, "p95_ms": 85.457}}, "truncated_count": 0}, "processing_total": {"avg_ms": 95.924, "count": 86, "max_ms": 151.278, "p95_ms": 134.485}, "read_interval": {"avg_ms": 348.421, "count": 86, "max_ms": 20554.218, "p95_ms": 148.656}, "triton_wait": {"avg_ms": 75.762, "count": 86, "max_ms": 89.458, "p95_ms": 86.662}, "visualize": {"avg_ms": 4.097, "count": 86, "max_ms": 6.113, "p95_ms": 5.277}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.918, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1835}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4218.938, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T00:45:17+09:00", "last_error_summary": "[h264 @ 0x5617f61a36c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5617f61a36c0] error while decoding MB 0 9, bytestream 6253 [h264 @ 0x5617f627c240] error while decoding MB 13 10, bytestream -13 [h264 @ 0x5617f61a36c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5617f61a36c0] error while decoding MB 0 5, bytestream 912 [h264 @ 0x5617f6254040] reference picture missing during reorder [h264 @ 0x5617f6254040] Missing reference picture, default is 65557 [h26", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 181, "h264_decode_error_count": 267, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:45:17+09:00", "last_error_summary": "[h264 @ 0x5617f61a36c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5617f61a36c0] error while decoding MB 0 9, bytestream 6253 [h264 @ 0x5617f627c240] error while decoding MB 13 10, bytestream -13 [h264 @ 0x5617f61a36c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5617f61a36c0] error while decoding MB 0 5, bytestream 912 [h264 @ 0x5617f6254040] reference picture missing during reorder [h264 @ 0x5617f6254040] Missing reference picture, default is 65557 [h26", "last_exit_at": "2026-07-02T00:45:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 481.136, "last_started_at": "2026-07-02T00:37:16+09:00", "pid": 31369, "process": {"exists": false, "pid": 31369}, "restart_count": 206, "restart_count_per_hour": 9, "rtsp_timeout_count": 50, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 207}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 136.58, "sample_count": 2580, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 220244, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:45:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T00:45:17+09:00", "last_frame_read_age_sec": 0.107, "last_frame_read_at": "2026-07-02T00:45:17+09:00", "mjpeg": {"age_sec": 0.104, "exists": true, "mtime": "2026-07-02T00:45:17.390469+09:00", "mtime_epoch": 1782920717.390469, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 127937}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:45:17+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 972.41, "name": "celery", "pid": 237, "rss_mb": 969.215, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 87, "pushed_frames": 87, "read_frames": 86, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.904, "pushed": 2.904, "read": 2.87}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.07, "count": 86, "max_ms": 4.252, "p95_ms": 4.005}, "gstreamer_push": {"avg_ms": 1.576, "count": 87, "max_ms": 6.47, "p95_ms": 3.935}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.388, "count": 86, "max_ms": 77.735, "p95_ms": 69.763}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.951, "count": 86, "max_ms": 88.697, "p95_ms": 83.336}, "yolo11_640_fire_11class": {"avg_ms": 67.622, "count": 86, "max_ms": 78.597, "p95_ms": 77.042}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.295, "count": 86, "max_ms": 89.445, "p95_ms": 86.65}}, "truncated_count": 0}, "processing_total": {"avg_ms": 95.949, "count": 87, "max_ms": 151.278, "p95_ms": 134.485}, "read_interval": {"avg_ms": 348.421, "count": 86, "max_ms": 20554.218, "p95_ms": 148.656}, "triton_wait": {"avg_ms": 76.104, "count": 86, "max_ms": 89.458, "p95_ms": 86.689}, "visualize": {"avg_ms": 4.079, "count": 86, "max_ms": 6.113, "p95_ms": 5.277}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.962, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1835}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4218.855, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 181, "h264_decode_error_count": 267, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:45:17+09:00", "last_error_summary": "[h264 @ 0x5617f61a36c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5617f61a36c0] error while decoding MB 0 9, bytestream 6253 [h264 @ 0x5617f627c240] error while decoding MB 13 10, bytestream -13 [h264 @ 0x5617f61a36c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5617f61a36c0] error while decoding MB 0 5, bytestream 912 [h264 @ 0x5617f6254040] reference picture missing during reorder [h264 @ 0x5617f6254040] Missing reference picture, default is 65557 [h26", "last_exit_at": "2026-07-02T00:45:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 481.144, "last_started_at": "2026-07-02T00:37:16+09:00", "pid": 31369, "process": {"exists": false, "pid": 31369}, "restart_count": 206, "restart_count_per_hour": 9, "rtsp_timeout_count": 50, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 207}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 136.58, "sample_count": 2580, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 220244, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:45:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-02T00:45:17+09:00", "last_frame_read_age_sec": 0.115, "last_frame_read_at": "2026-07-02T00:45:17+09:00", "mjpeg": {"age_sec": 0.112, "exists": true, "mtime": "2026-07-02T00:45:17.390469+09:00", "mtime_epoch": 1782920717.390469, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 127937}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:45:17+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 972.41, "name": "celery", "pid": 237, "rss_mb": 969.215, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 87, "pushed_frames": 87, "read_frames": 86, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.903, "pushed": 2.903, "read": 2.87}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.07, "count": 86, "max_ms": 4.252, "p95_ms": 4.005}, "gstreamer_push": {"avg_ms": 1.576, "count": 87, "max_ms": 6.47, "p95_ms": 3.935}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.388, "count": 86, "max_ms": 77.735, "p95_ms": 69.763}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.951, "count": 86, "max_ms": 88.697, "p95_ms": 83.336}, "yolo11_640_fire_11class": {"avg_ms": 67.622, "count": 86, "max_ms": 78.597, "p95_ms": 77.042}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.295, "count": 86, "max_ms": 89.445, "p95_ms": 86.65}}, "truncated_count": 0}, "processing_total": {"avg_ms": 95.949, "count": 87, "max_ms": 151.278, "p95_ms": 134.485}, "read_interval": {"avg_ms": 348.421, "count": 86, "max_ms": 20554.218, "p95_ms": 148.656}, "triton_wait": {"avg_ms": 76.104, "count": 86, "max_ms": 89.458, "p95_ms": 86.689}, "visualize": {"avg_ms": 4.079, "count": 86, "max_ms": 6.113, "p95_ms": 5.277}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.97, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1835}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4329.637, "limit_mb": 8796093022207.996}, "detail": {"pid": 31476, "restart_count": 207, "start_count": 208}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 181, "h264_decode_error_count": 267, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:45:17+09:00", "last_error_summary": "[h264 @ 0x5617f61a36c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5617f61a36c0] error while decoding MB 0 9, bytestream 6253 [h264 @ 0x5617f627c240] error while decoding MB 13 10, bytestream -13 [h264 @ 0x5617f61a36c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5617f61a36c0] error while decoding MB 0 5, bytestream 912 [h264 @ 0x5617f6254040] reference picture missing during reorder [h264 @ 0x5617f6254040] Missing reference picture, default is 65557 [h26", "last_exit_at": "2026-07-02T00:45:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T00:45:19+09:00", "pid": 31476, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.301, "name": "ffmpeg", "pid": 31476, "rss_mb": 139.301, "state": "S (sleeping)", "threads": 106}, "restart_count": 207, "restart_count_per_hour": 10, "rtsp_timeout_count": 50, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 208}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 136.575, "sample_count": 2541, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 220244, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:45:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.217, "last_frame_pushed_at": "2026-07-02T00:45:17+09:00", "last_frame_read_age_sec": 4.317, "last_frame_read_at": "2026-07-02T00:45:17+09:00", "mjpeg": {"age_sec": 4.199, "exists": true, "mtime": "2026-07-02T00:45:17.506471+09:00", "mtime_epoch": 1782920717.5064707, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 118570}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:45:21+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 972.41, "name": "celery", "pid": 237, "rss_mb": 969.215, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 53, "pushed_frames": 53, "read_frames": 53, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.39, "pushed": 5.39, "read": 5.39}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.64, "count": 53, "max_ms": 3.603, "p95_ms": 2.012}, "gstreamer_push": {"avg_ms": 1.701, "count": 53, "max_ms": 6.47, "p95_ms": 4.024}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.006, "count": 53, "max_ms": 77.735, "p95_ms": 76.656}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.34, "count": 53, "max_ms": 86.676, "p95_ms": 79.985}, "yolo11_640_fire_11class": {"avg_ms": 70.939, "count": 53, "max_ms": 78.326, "p95_ms": 77.042}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.418, "count": 53, "max_ms": 87.928, "p95_ms": 85.377}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.002, "count": 53, "max_ms": 136.61, "p95_ms": 98.092}, "read_interval": {"avg_ms": 491.885, "count": 53, "max_ms": 20554.218, "p95_ms": 116.956}, "triton_wait": {"avg_ms": 78.937, "count": 53, "max_ms": 87.94, "p95_ms": 86.662}, "visualize": {"avg_ms": 3.799, "count": 53, "max_ms": 5.099, "p95_ms": 4.166}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 9.833, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1856}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4217.566, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5609af1bf100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5609af1bf100] error while decoding MB 0 10, bytestream 7415 [h264 @ 0x5609aee83700] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5609aee83700] error while decoding MB 0 3, bytestream 5664 [h264 @ 0x5609af021400] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5609af021400] error while decoding MB 0 3, bytestream 9039 [h264 @ 0x5609aee83700] error while decoding MB 27 4, byte", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 182, "h264_decode_error_count": 271, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:50:37+09:00", "last_error_summary": "[h264 @ 0x5609af1bf100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5609af1bf100] error while decoding MB 0 10, bytestream 7415 [h264 @ 0x5609aee83700] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5609aee83700] error while decoding MB 0 3, bytestream 5664 [h264 @ 0x5609af021400] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5609af021400] error while decoding MB 0 3, bytestream 9039 [h264 @ 0x5609aee83700] error while decoding MB 27 4, byte", "last_exit_at": "2026-07-02T00:45:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 318.237, "last_started_at": "2026-07-02T00:45:19+09:00", "pid": 31476, "process": {"exists": false, "pid": 31476}, "restart_count": 207, "restart_count_per_hour": 10, "rtsp_timeout_count": 52, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 208}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.143, "sample_count": 2429, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 222820, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:50:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.068, "last_frame_pushed_at": "2026-07-02T00:50:37+09:00", "last_frame_read_age_sec": 0.056, "last_frame_read_at": "2026-07-02T00:50:37+09:00", "mjpeg": {"age_sec": 0.05, "exists": true, "mtime": "2026-07-02T00:50:37.891082+09:00", "mtime_epoch": 1782921037.8910818, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 133220}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:50:37+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 972.41, "name": "celery", "pid": 237, "rss_mb": 964.262, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 93, "pushed_frames": 93, "read_frames": 93, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.103, "pushed": 3.103, "read": 3.103}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.011, "count": 92, "max_ms": 5.195, "p95_ms": 4.503}, "gstreamer_push": {"avg_ms": 1.418, "count": 93, "max_ms": 6.266, "p95_ms": 2.699}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.508, "count": 92, "max_ms": 73.875, "p95_ms": 66.502}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.395, "count": 92, "max_ms": 87.125, "p95_ms": 85.284}, "yolo11_640_fire_11class": {"avg_ms": 62.31, "count": 92, "max_ms": 81.143, "p95_ms": 76.586}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.484, "count": 92, "max_ms": 88.491, "p95_ms": 87.427}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.603, "count": 93, "max_ms": 144.568, "p95_ms": 135.949}, "read_interval": {"avg_ms": 323.058, "count": 93, "max_ms": 19177.662, "p95_ms": 149.619}, "triton_wait": {"avg_ms": 71.01, "count": 92, "max_ms": 88.503, "p95_ms": 87.44}, "visualize": {"avg_ms": 4.524, "count": 92, "max_ms": 6.214, "p95_ms": 5.397}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.97, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1856}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4216.871, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T00:50:37+09:00", "last_error_summary": "[h264 @ 0x5609af1bf100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5609af1bf100] error while decoding MB 0 10, bytestream 7415 [h264 @ 0x5609aee83700] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5609aee83700] error while decoding MB 0 3, bytestream 5664 [h264 @ 0x5609af021400] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5609af021400] error while decoding MB 0 3, bytestream 9039 [h264 @ 0x5609aee83700] error while decoding MB 27 4, byte", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 182, "h264_decode_error_count": 271, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:50:37+09:00", "last_error_summary": "[h264 @ 0x5609af1bf100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5609af1bf100] error while decoding MB 0 10, bytestream 7415 [h264 @ 0x5609aee83700] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5609aee83700] error while decoding MB 0 3, bytestream 5664 [h264 @ 0x5609af021400] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5609af021400] error while decoding MB 0 3, bytestream 9039 [h264 @ 0x5609aee83700] error while decoding MB 27 4, byte", "last_exit_at": "2026-07-02T00:50:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 318.286, "last_started_at": "2026-07-02T00:45:19+09:00", "pid": 31476, "process": {"exists": false, "pid": 31476}, "restart_count": 207, "restart_count_per_hour": 10, "rtsp_timeout_count": 52, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 208}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.143, "sample_count": 2430, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 222821, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:50:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-02T00:50:37+09:00", "last_frame_read_age_sec": 0.105, "last_frame_read_at": "2026-07-02T00:50:37+09:00", "mjpeg": {"age_sec": 0.099, "exists": true, "mtime": "2026-07-02T00:50:37.891082+09:00", "mtime_epoch": 1782921037.8910818, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 133220}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:50:37+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 972.41, "name": "celery", "pid": 237, "rss_mb": 964.262, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 93, "pushed_frames": 93, "read_frames": 92, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.111, "pushed": 3.111, "read": 3.078}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.002, "count": 93, "max_ms": 5.195, "p95_ms": 4.503}, "gstreamer_push": {"avg_ms": 1.419, "count": 93, "max_ms": 6.266, "p95_ms": 2.699}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.572, "count": 93, "max_ms": 73.875, "p95_ms": 66.502}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.482, "count": 93, "max_ms": 87.125, "p95_ms": 85.284}, "yolo11_640_fire_11class": {"avg_ms": 62.47, "count": 93, "max_ms": 81.143, "p95_ms": 77.2}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.63, "count": 93, "max_ms": 88.491, "p95_ms": 87.427}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.221, "count": 93, "max_ms": 144.568, "p95_ms": 135.949}, "read_interval": {"avg_ms": 324.973, "count": 92, "max_ms": 19177.662, "p95_ms": 149.619}, "triton_wait": {"avg_ms": 71.14, "count": 93, "max_ms": 88.503, "p95_ms": 87.44}, "visualize": {"avg_ms": 4.526, "count": 93, "max_ms": 6.214, "p95_ms": 5.397}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.894, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1856}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4217.027, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 182, "h264_decode_error_count": 271, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:50:37+09:00", "last_error_summary": "[h264 @ 0x5609af1bf100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5609af1bf100] error while decoding MB 0 10, bytestream 7415 [h264 @ 0x5609aee83700] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5609aee83700] error while decoding MB 0 3, bytestream 5664 [h264 @ 0x5609af021400] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5609af021400] error while decoding MB 0 3, bytestream 9039 [h264 @ 0x5609aee83700] error while decoding MB 27 4, byte", "last_exit_at": "2026-07-02T00:50:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 318.295, "last_started_at": "2026-07-02T00:45:19+09:00", "pid": 31476, "process": {"exists": false, "pid": 31476}, "restart_count": 207, "restart_count_per_hour": 10, "rtsp_timeout_count": 52, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 208}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.143, "sample_count": 2430, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 222821, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:50:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.017, "last_frame_pushed_at": "2026-07-02T00:50:37+09:00", "last_frame_read_age_sec": 0.115, "last_frame_read_at": "2026-07-02T00:50:37+09:00", "mjpeg": {"age_sec": 0.109, "exists": true, "mtime": "2026-07-02T00:50:37.891082+09:00", "mtime_epoch": 1782921037.8910818, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 133220}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:50:38+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 972.41, "name": "celery", "pid": 237, "rss_mb": 964.262, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 93, "pushed_frames": 93, "read_frames": 92, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.11, "pushed": 3.11, "read": 3.077}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.002, "count": 93, "max_ms": 5.195, "p95_ms": 4.503}, "gstreamer_push": {"avg_ms": 1.419, "count": 93, "max_ms": 6.266, "p95_ms": 2.699}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.572, "count": 93, "max_ms": 73.875, "p95_ms": 66.502}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.482, "count": 93, "max_ms": 87.125, "p95_ms": 85.284}, "yolo11_640_fire_11class": {"avg_ms": 62.47, "count": 93, "max_ms": 81.143, "p95_ms": 77.2}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.63, "count": 93, "max_ms": 88.491, "p95_ms": 87.427}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.221, "count": 93, "max_ms": 144.568, "p95_ms": 135.949}, "read_interval": {"avg_ms": 324.973, "count": 92, "max_ms": 19177.662, "p95_ms": 149.619}, "triton_wait": {"avg_ms": 71.14, "count": 93, "max_ms": 88.503, "p95_ms": 87.44}, "visualize": {"avg_ms": 4.526, "count": 93, "max_ms": 6.214, "p95_ms": 5.397}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.903, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1856}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4324.781, "limit_mb": 8796093022207.996}, "detail": {"pid": 31583, "restart_count": 208, "start_count": 209}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 182, "h264_decode_error_count": 271, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:50:37+09:00", "last_error_summary": "[h264 @ 0x5609af1bf100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5609af1bf100] error while decoding MB 0 10, bytestream 7415 [h264 @ 0x5609aee83700] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5609aee83700] error while decoding MB 0 3, bytestream 5664 [h264 @ 0x5609af021400] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5609af021400] error while decoding MB 0 3, bytestream 9039 [h264 @ 0x5609aee83700] error while decoding MB 27 4, byte", "last_exit_at": "2026-07-02T00:50:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T00:50:40+09:00", "pid": 31583, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.934, "name": "ffmpeg", "pid": 31583, "rss_mb": 138.934, "state": "S (sleeping)", "threads": 106}, "restart_count": 208, "restart_count_per_hour": 11, "rtsp_timeout_count": 52, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 209}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.089, "sample_count": 2397, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 222821, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:50:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.079, "last_frame_pushed_at": "2026-07-02T00:50:37+09:00", "last_frame_read_age_sec": 4.176, "last_frame_read_at": "2026-07-02T00:50:37+09:00", "mjpeg": {"age_sec": 4.062, "exists": true, "mtime": "2026-07-02T00:50:37.999083+09:00", "mtime_epoch": 1782921037.9990833, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 134083}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:50:42+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 972.41, "name": "celery", "pid": 237, "rss_mb": 964.262, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 62, "pushed_frames": 62, "read_frames": 61, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.076, "pushed": 2.076, "read": 2.042}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.508, "count": 62, "max_ms": 5.195, "p95_ms": 4.503}, "gstreamer_push": {"avg_ms": 1.521, "count": 62, "max_ms": 6.266, "p95_ms": 3.581}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.944, "count": 62, "max_ms": 71.254, "p95_ms": 66.023}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.791, "count": 62, "max_ms": 87.125, "p95_ms": 85.284}, "yolo11_640_fire_11class": {"avg_ms": 63.282, "count": 62, "max_ms": 81.143, "p95_ms": 77.2}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.963, "count": 62, "max_ms": 88.078, "p95_ms": 85.097}}, "truncated_count": 0}, "processing_total": {"avg_ms": 95.78, "count": 62, "max_ms": 136.56, "p95_ms": 131.65}, "read_interval": {"avg_ms": 423.39, "count": 61, "max_ms": 19177.662, "p95_ms": 147.908}, "triton_wait": {"avg_ms": 72.203, "count": 62, "max_ms": 88.09, "p95_ms": 86.447}, "visualize": {"avg_ms": 4.231, "count": 62, "max_ms": 5.734, "p95_ms": 5.226}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.871, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1875}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4224.852, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 9, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55f70124b740] top block unavailable for requested intra mode -1 [h264 @ 0x55f70124b740] error while decoding MB 79 0, bytestream 8154 [h264 @ 0x55f701d2f240] left block unavailable for requested intra mode [h264 @ 0x55f701d2f240] error while decoding MB 0 23, bytestream 1287 [h264 @ 0x55f701b91540] cabac decode of qscale diff failed at 79 22 [h264 @ 0x55f701b91540] error while decoding MB 79 22, bytestream 1371 [h264 @ 0x55f701dfe0c0] left block unavailable for requested intra4x4 mode ", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 183, "h264_decode_error_count": 280, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:55:10+09:00", "last_error_summary": "[h264 @ 0x55f70124b740] top block unavailable for requested intra mode -1 [h264 @ 0x55f70124b740] error while decoding MB 79 0, bytestream 8154 [h264 @ 0x55f701d2f240] left block unavailable for requested intra mode [h264 @ 0x55f701d2f240] error while decoding MB 0 23, bytestream 1287 [h264 @ 0x55f701b91540] cabac decode of qscale diff failed at 79 22 [h264 @ 0x55f701b91540] error while decoding MB 79 22, bytestream 1371 [h264 @ 0x55f701dfe0c0] left block unavailable for requested intra4x4 mode ", "last_exit_at": "2026-07-02T00:50:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 270.625, "last_started_at": "2026-07-02T00:50:40+09:00", "pid": 31583, "process": {"exists": false, "pid": 31583}, "restart_count": 208, "restart_count_per_hour": 10, "rtsp_timeout_count": 54, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 209}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.603, "sample_count": 2257, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 225005, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:55:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.088, "last_frame_pushed_at": "2026-07-02T00:55:10+09:00", "last_frame_read_age_sec": 0.076, "last_frame_read_at": "2026-07-02T00:55:10+09:00", "mjpeg": {"age_sec": 0.07, "exists": true, "mtime": "2026-07-02T00:55:10.615074+09:00", "mtime_epoch": 1782921310.6150737, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 116650}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:55:10+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 972.41, "name": "celery", "pid": 237, "rss_mb": 968.609, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 89, "pushed_frames": 89, "read_frames": 89, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.972, "pushed": 2.972, "read": 2.972}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.909, "count": 89, "max_ms": 5.352, "p95_ms": 3.879}, "gstreamer_push": {"avg_ms": 1.599, "count": 89, "max_ms": 6.006, "p95_ms": 3.797}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.552, "count": 89, "max_ms": 78.055, "p95_ms": 75.913}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.943, "count": 89, "max_ms": 86.375, "p95_ms": 80.341}, "yolo11_640_fire_11class": {"avg_ms": 70.639, "count": 89, "max_ms": 78.383, "p95_ms": 76.628}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.791, "count": 89, "max_ms": 92.499, "p95_ms": 88.544}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.618, "count": 89, "max_ms": 136.301, "p95_ms": 123.341}, "read_interval": {"avg_ms": 336.646, "count": 89, "max_ms": 20143.936, "p95_ms": 142.466}, "triton_wait": {"avg_ms": 80.917, "count": 89, "max_ms": 92.511, "p95_ms": 88.556}, "visualize": {"avg_ms": 3.937, "count": 89, "max_ms": 8.58, "p95_ms": 5.072}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.944, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1875}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4224.812, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T00:55:10+09:00", "last_error_summary": "[h264 @ 0x55f70124b740] top block unavailable for requested intra mode -1 [h264 @ 0x55f70124b740] error while decoding MB 79 0, bytestream 8154 [h264 @ 0x55f701d2f240] left block unavailable for requested intra mode [h264 @ 0x55f701d2f240] error while decoding MB 0 23, bytestream 1287 [h264 @ 0x55f701b91540] cabac decode of qscale diff failed at 79 22 [h264 @ 0x55f701b91540] error while decoding MB 79 22, bytestream 1371 [h264 @ 0x55f701dfe0c0] left block unavailable for requested intra4x4 mode ", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 183, "h264_decode_error_count": 280, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:55:10+09:00", "last_error_summary": "[h264 @ 0x55f70124b740] top block unavailable for requested intra mode -1 [h264 @ 0x55f70124b740] error while decoding MB 79 0, bytestream 8154 [h264 @ 0x55f701d2f240] left block unavailable for requested intra mode [h264 @ 0x55f701d2f240] error while decoding MB 0 23, bytestream 1287 [h264 @ 0x55f701b91540] cabac decode of qscale diff failed at 79 22 [h264 @ 0x55f701b91540] error while decoding MB 79 22, bytestream 1371 [h264 @ 0x55f701dfe0c0] left block unavailable for requested intra4x4 mode ", "last_exit_at": "2026-07-02T00:55:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 270.652, "last_started_at": "2026-07-02T00:50:40+09:00", "pid": 31583, "process": {"exists": false, "pid": 31583}, "restart_count": 208, "restart_count_per_hour": 10, "rtsp_timeout_count": 54, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 209}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.603, "sample_count": 2258, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 225006, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:55:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-02T00:55:10+09:00", "last_frame_read_age_sec": 0.103, "last_frame_read_at": "2026-07-02T00:55:10+09:00", "mjpeg": {"age_sec": 0.097, "exists": true, "mtime": "2026-07-02T00:55:10.615074+09:00", "mtime_epoch": 1782921310.6150737, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 116650}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:55:10+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 972.41, "name": "celery", "pid": 237, "rss_mb": 968.609, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 90, "pushed_frames": 90, "read_frames": 89, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.003, "pushed": 3.003, "read": 2.97}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.908, "count": 90, "max_ms": 5.352, "p95_ms": 3.879}, "gstreamer_push": {"avg_ms": 1.598, "count": 90, "max_ms": 6.006, "p95_ms": 3.797}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.457, "count": 90, "max_ms": 78.055, "p95_ms": 75.913}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.019, "count": 90, "max_ms": 86.375, "p95_ms": 80.341}, "yolo11_640_fire_11class": {"avg_ms": 70.68, "count": 90, "max_ms": 78.383, "p95_ms": 76.628}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.786, "count": 90, "max_ms": 92.499, "p95_ms": 88.544}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.568, "count": 90, "max_ms": 136.301, "p95_ms": 123.341}, "read_interval": {"avg_ms": 336.646, "count": 89, "max_ms": 20143.936, "p95_ms": 142.466}, "triton_wait": {"avg_ms": 80.91, "count": 90, "max_ms": 92.511, "p95_ms": 88.556}, "visualize": {"avg_ms": 3.946, "count": 90, "max_ms": 8.58, "p95_ms": 5.072}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.971, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1875}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4224.707, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 183, "h264_decode_error_count": 280, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:55:10+09:00", "last_error_summary": "[h264 @ 0x55f70124b740] top block unavailable for requested intra mode -1 [h264 @ 0x55f70124b740] error while decoding MB 79 0, bytestream 8154 [h264 @ 0x55f701d2f240] left block unavailable for requested intra mode [h264 @ 0x55f701d2f240] error while decoding MB 0 23, bytestream 1287 [h264 @ 0x55f701b91540] cabac decode of qscale diff failed at 79 22 [h264 @ 0x55f701b91540] error while decoding MB 79 22, bytestream 1371 [h264 @ 0x55f701dfe0c0] left block unavailable for requested intra4x4 mode ", "last_exit_at": "2026-07-02T00:55:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 270.661, "last_started_at": "2026-07-02T00:50:40+09:00", "pid": 31583, "process": {"exists": false, "pid": 31583}, "restart_count": 208, "restart_count_per_hour": 10, "rtsp_timeout_count": 54, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 209}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.603, "sample_count": 2258, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 225006, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:55:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.018, "last_frame_pushed_at": "2026-07-02T00:55:10+09:00", "last_frame_read_age_sec": 0.112, "last_frame_read_at": "2026-07-02T00:55:10+09:00", "mjpeg": {"age_sec": 0.106, "exists": true, "mtime": "2026-07-02T00:55:10.615074+09:00", "mtime_epoch": 1782921310.6150737, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 116650}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:55:10+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 972.41, "name": "celery", "pid": 237, "rss_mb": 968.609, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 90, "pushed_frames": 90, "read_frames": 89, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.002, "pushed": 3.002, "read": 2.969}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.908, "count": 90, "max_ms": 5.352, "p95_ms": 3.879}, "gstreamer_push": {"avg_ms": 1.598, "count": 90, "max_ms": 6.006, "p95_ms": 3.797}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.457, "count": 90, "max_ms": 78.055, "p95_ms": 75.913}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.019, "count": 90, "max_ms": 86.375, "p95_ms": 80.341}, "yolo11_640_fire_11class": {"avg_ms": 70.68, "count": 90, "max_ms": 78.383, "p95_ms": 76.628}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.786, "count": 90, "max_ms": 92.499, "p95_ms": 88.544}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.568, "count": 90, "max_ms": 136.301, "p95_ms": 123.341}, "read_interval": {"avg_ms": 336.646, "count": 89, "max_ms": 20143.936, "p95_ms": 142.466}, "triton_wait": {"avg_ms": 80.91, "count": 90, "max_ms": 92.511, "p95_ms": 88.556}, "visualize": {"avg_ms": 3.946, "count": 90, "max_ms": 8.58, "p95_ms": 5.072}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.98, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1875}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4332.578, "limit_mb": 8796093022207.996}, "detail": {"pid": 31690, "restart_count": 209, "start_count": 210}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 183, "h264_decode_error_count": 280, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T00:55:10+09:00", "last_error_summary": "[h264 @ 0x55f70124b740] top block unavailable for requested intra mode -1 [h264 @ 0x55f70124b740] error while decoding MB 79 0, bytestream 8154 [h264 @ 0x55f701d2f240] left block unavailable for requested intra mode [h264 @ 0x55f701d2f240] error while decoding MB 0 23, bytestream 1287 [h264 @ 0x55f701b91540] cabac decode of qscale diff failed at 79 22 [h264 @ 0x55f701b91540] error while decoding MB 79 22, bytestream 1371 [h264 @ 0x55f701dfe0c0] left block unavailable for requested intra4x4 mode ", "last_exit_at": "2026-07-02T00:55:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T00:55:13+09:00", "pid": 31690, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.168, "name": "ffmpeg", "pid": 31690, "rss_mb": 139.168, "state": "S (sleeping)", "threads": 106}, "restart_count": 209, "restart_count_per_hour": 11, "rtsp_timeout_count": 54, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 210}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.615, "sample_count": 2220, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 225006, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T00:55:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.593, "last_frame_pushed_at": "2026-07-02T00:55:10+09:00", "last_frame_read_age_sec": 4.688, "last_frame_read_at": "2026-07-02T00:55:10+09:00", "mjpeg": {"age_sec": 4.577, "exists": true, "mtime": "2026-07-02T00:55:10.719075+09:00", "mtime_epoch": 1782921310.7190752, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 117198}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T00:55:15+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 972.41, "name": "celery", "pid": 237, "rss_mb": 968.609, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 70, "pushed_frames": 70, "read_frames": 70, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.692, "pushed": 5.692, "read": 5.692}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.779, "count": 70, "max_ms": 5.352, "p95_ms": 3.678}, "gstreamer_push": {"avg_ms": 1.669, "count": 70, "max_ms": 6.006, "p95_ms": 3.822}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.712, "count": 70, "max_ms": 78.055, "p95_ms": 75.999}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.355, "count": 70, "max_ms": 86.375, "p95_ms": 79.715}, "yolo11_640_fire_11class": {"avg_ms": 71.404, "count": 70, "max_ms": 78.383, "p95_ms": 76.856}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.22, "count": 70, "max_ms": 92.499, "p95_ms": 88.728}}, "truncated_count": 0}, "processing_total": {"avg_ms": 95.498, "count": 70, "max_ms": 123.341, "p95_ms": 103.089}, "read_interval": {"avg_ms": 396.5, "count": 70, "max_ms": 20143.936, "p95_ms": 125.291}, "triton_wait": {"avg_ms": 82.329, "count": 70, "max_ms": 92.511, "p95_ms": 88.74}, "visualize": {"avg_ms": 3.898, "count": 70, "max_ms": 8.58, "p95_ms": 4.762}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 12.299, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1896}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4228.332, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 2, "sample_message": "[rtsp @ 0x56330d17ef00] RTP: PT=60: bad cseq 7ea0 expected=7276 [h264 @ 0x56330dd94d80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56330dd94d80] error while decoding MB 0 15, bytestream 704 [h264 @ 0x56330d2c61c0] reference picture missing during reorder [h264 @ 0x56330d2c61c0] Missing reference picture, default is 65551 [h264 @ 0x56330d2da380] mmco: unref short failure [h264 @ 0x56330d2da380] illegal short term buffer state detected [h264 @ 0x56330de63c00] error while decod", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 184, "h264_decode_error_count": 283, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T01:00:21+09:00", "last_error_summary": "[rtsp @ 0x56330d17ef00] RTP: PT=60: bad cseq 7ea0 expected=7276 [h264 @ 0x56330dd94d80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56330dd94d80] error while decoding MB 0 15, bytestream 704 [h264 @ 0x56330d2c61c0] reference picture missing during reorder [h264 @ 0x56330d2c61c0] Missing reference picture, default is 65551 [h264 @ 0x56330d2da380] mmco: unref short failure [h264 @ 0x56330d2da380] illegal short term buffer state detected [h264 @ 0x56330de63c00] error while decod", "last_exit_at": "2026-07-02T00:55:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 308.072, "last_started_at": "2026-07-02T00:55:13+09:00", "pid": 31690, "process": {"exists": false, "pid": 31690}, "restart_count": 209, "restart_count_per_hour": 11, "rtsp_timeout_count": 56, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 210}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 136.992, "sample_count": 2550, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 227608, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T01:00:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.066, "last_frame_pushed_at": "2026-07-02T01:00:21+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-02T01:00:21+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-02T01:00:21.319663+09:00", "mtime_epoch": 1782921621.3196633, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 129577}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T01:00:21+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 973.602, "name": "celery", "pid": 237, "rss_mb": 969.992, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 103, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.436, "pushed": 3.436, "read": 3.436}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.861, "count": 103, "max_ms": 6.214, "p95_ms": 4.094}, "gstreamer_push": {"avg_ms": 1.486, "count": 103, "max_ms": 6.276, "p95_ms": 2.419}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.798, "count": 103, "max_ms": 78.427, "p95_ms": 74.87}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.849, "count": 103, "max_ms": 85.107, "p95_ms": 80.648}, "yolo11_640_fire_11class": {"avg_ms": 59.712, "count": 103, "max_ms": 78.893, "p95_ms": 77.068}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.07, "count": 103, "max_ms": 91.662, "p95_ms": 88.02}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.32, "count": 103, "max_ms": 153.679, "p95_ms": 108.142}, "read_interval": {"avg_ms": 291.485, "count": 103, "max_ms": 20139.61, "p95_ms": 124.372}, "triton_wait": {"avg_ms": 66.731, "count": 103, "max_ms": 91.673, "p95_ms": 88.031}, "visualize": {"avg_ms": 3.944, "count": 103, "max_ms": 9.346, "p95_ms": 5.242}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.979, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1896}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4227.812, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T01:00:21+09:00", "last_error_summary": "[rtsp @ 0x56330d17ef00] RTP: PT=60: bad cseq 7ea0 expected=7276 [h264 @ 0x56330dd94d80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56330dd94d80] error while decoding MB 0 15, bytestream 704 [h264 @ 0x56330d2c61c0] reference picture missing during reorder [h264 @ 0x56330d2c61c0] Missing reference picture, default is 65551 [h264 @ 0x56330d2da380] mmco: unref short failure [h264 @ 0x56330d2da380] illegal short term buffer state detected [h264 @ 0x56330de63c00] error while decod", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 184, "h264_decode_error_count": 283, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T01:00:21+09:00", "last_error_summary": "[rtsp @ 0x56330d17ef00] RTP: PT=60: bad cseq 7ea0 expected=7276 [h264 @ 0x56330dd94d80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56330dd94d80] error while decoding MB 0 15, bytestream 704 [h264 @ 0x56330d2c61c0] reference picture missing during reorder [h264 @ 0x56330d2c61c0] Missing reference picture, default is 65551 [h264 @ 0x56330d2da380] mmco: unref short failure [h264 @ 0x56330d2da380] illegal short term buffer state detected [h264 @ 0x56330de63c00] error while decod", "last_exit_at": "2026-07-02T01:00:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 308.116, "last_started_at": "2026-07-02T00:55:13+09:00", "pid": 31690, "process": {"exists": false, "pid": 31690}, "restart_count": 209, "restart_count_per_hour": 11, "rtsp_timeout_count": 56, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 210}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 136.992, "sample_count": 2550, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 227609, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T01:00:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T01:00:21+09:00", "last_frame_read_age_sec": 0.098, "last_frame_read_at": "2026-07-02T01:00:21+09:00", "mjpeg": {"age_sec": 0.092, "exists": true, "mtime": "2026-07-02T01:00:21.319663+09:00", "mtime_epoch": 1782921621.3196633, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 129577}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T01:00:21+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 973.602, "name": "celery", "pid": 237, "rss_mb": 969.992, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.444, "pushed": 3.444, "read": 3.41}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.861, "count": 103, "max_ms": 6.214, "p95_ms": 4.094}, "gstreamer_push": {"avg_ms": 1.487, "count": 103, "max_ms": 6.276, "p95_ms": 2.419}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.627, "count": 103, "max_ms": 78.427, "p95_ms": 70.394}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.878, "count": 103, "max_ms": 85.107, "p95_ms": 80.648}, "yolo11_640_fire_11class": {"avg_ms": 59.765, "count": 103, "max_ms": 79.439, "p95_ms": 77.405}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.859, "count": 103, "max_ms": 91.662, "p95_ms": 87.703}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.236, "count": 103, "max_ms": 153.679, "p95_ms": 108.142}, "read_interval": {"avg_ms": 293.218, "count": 102, "max_ms": 20139.61, "p95_ms": 124.372}, "triton_wait": {"avg_ms": 66.648, "count": 103, "max_ms": 91.673, "p95_ms": 87.715}, "visualize": {"avg_ms": 3.944, "count": 103, "max_ms": 9.346, "p95_ms": 5.242}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.909, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1896}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4227.863, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 184, "h264_decode_error_count": 283, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T01:00:21+09:00", "last_error_summary": "[rtsp @ 0x56330d17ef00] RTP: PT=60: bad cseq 7ea0 expected=7276 [h264 @ 0x56330dd94d80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56330dd94d80] error while decoding MB 0 15, bytestream 704 [h264 @ 0x56330d2c61c0] reference picture missing during reorder [h264 @ 0x56330d2c61c0] Missing reference picture, default is 65551 [h264 @ 0x56330d2da380] mmco: unref short failure [h264 @ 0x56330d2da380] illegal short term buffer state detected [h264 @ 0x56330de63c00] error while decod", "last_exit_at": "2026-07-02T01:00:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 308.124, "last_started_at": "2026-07-02T00:55:13+09:00", "pid": 31690, "process": {"exists": false, "pid": 31690}, "restart_count": 209, "restart_count_per_hour": 11, "rtsp_timeout_count": 56, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 210}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 136.992, "sample_count": 2550, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 227609, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T01:00:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-02T01:00:21+09:00", "last_frame_read_age_sec": 0.106, "last_frame_read_at": "2026-07-02T01:00:21+09:00", "mjpeg": {"age_sec": 0.099, "exists": true, "mtime": "2026-07-02T01:00:21.319663+09:00", "mtime_epoch": 1782921621.3196633, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 129577}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T01:00:21+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 973.602, "name": "celery", "pid": 237, "rss_mb": 969.992, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.443, "pushed": 3.443, "read": 3.409}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.861, "count": 103, "max_ms": 6.214, "p95_ms": 4.094}, "gstreamer_push": {"avg_ms": 1.487, "count": 103, "max_ms": 6.276, "p95_ms": 2.419}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.627, "count": 103, "max_ms": 78.427, "p95_ms": 70.394}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.878, "count": 103, "max_ms": 85.107, "p95_ms": 80.648}, "yolo11_640_fire_11class": {"avg_ms": 59.765, "count": 103, "max_ms": 79.439, "p95_ms": 77.405}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.859, "count": 103, "max_ms": 91.662, "p95_ms": 87.703}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.236, "count": 103, "max_ms": 153.679, "p95_ms": 108.142}, "read_interval": {"avg_ms": 293.218, "count": 102, "max_ms": 20139.61, "p95_ms": 124.372}, "triton_wait": {"avg_ms": 66.648, "count": 103, "max_ms": 91.673, "p95_ms": 87.715}, "visualize": {"avg_ms": 3.944, "count": 103, "max_ms": 9.346, "p95_ms": 5.242}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.917, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1896}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4330.578, "limit_mb": 8796093022207.996}, "detail": {"pid": 31797, "restart_count": 210, "start_count": 211}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 184, "h264_decode_error_count": 283, "h264_decode_error_count_per_hour": 40, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T01:00:21+09:00", "last_error_summary": "[rtsp @ 0x56330d17ef00] RTP: PT=60: bad cseq 7ea0 expected=7276 [h264 @ 0x56330dd94d80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56330dd94d80] error while decoding MB 0 15, bytestream 704 [h264 @ 0x56330d2c61c0] reference picture missing during reorder [h264 @ 0x56330d2c61c0] Missing reference picture, default is 65551 [h264 @ 0x56330d2da380] mmco: unref short failure [h264 @ 0x56330d2da380] illegal short term buffer state detected [h264 @ 0x56330de63c00] error while decod", "last_exit_at": "2026-07-02T01:00:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T01:00:24+09:00", "pid": 31797, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.375, "name": "ffmpeg", "pid": 31797, "rss_mb": 136.375, "state": "S (sleeping)", "threads": 106}, "restart_count": 210, "restart_count_per_hour": 12, "rtsp_timeout_count": 56, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 211}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 136.966, "sample_count": 2507, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 227609, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T01:00:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.821, "last_frame_pushed_at": "2026-07-02T01:00:21+09:00", "last_frame_read_age_sec": 4.913, "last_frame_read_at": "2026-07-02T01:00:21+09:00", "mjpeg": {"age_sec": 4.803, "exists": true, "mtime": "2026-07-02T01:00:21.423665+09:00", "mtime_epoch": 1782921621.4236648, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 130253}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T01:00:26+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 973.602, "name": "celery", "pid": 237, "rss_mb": 969.992, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 58, "pushed_frames": 58, "read_frames": 57, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.936, "pushed": 1.936, "read": 1.903}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.678, "count": 58, "max_ms": 4.244, "p95_ms": 2.023}, "gstreamer_push": {"avg_ms": 1.456, "count": 58, "max_ms": 3.912, "p95_ms": 2.984}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.288, "count": 58, "max_ms": 60.124, "p95_ms": 57.248}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.934, "count": 58, "max_ms": 85.107, "p95_ms": 80.648}, "yolo11_640_fire_11class": {"avg_ms": 55.299, "count": 58, "max_ms": 79.439, "p95_ms": 76.059}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.528, "count": 58, "max_ms": 84.222, "p95_ms": 83.959}}, "truncated_count": 0}, "processing_total": {"avg_ms": 72.546, "count": 58, "max_ms": 137.779, "p95_ms": 95.441}, "read_interval": {"avg_ms": 440.416, "count": 57, "max_ms": 20139.61, "p95_ms": 118.143}, "triton_wait": {"avg_ms": 60.753, "count": 58, "max_ms": 85.119, "p95_ms": 84.207}, "visualize": {"avg_ms": 3.717, "count": 58, "max_ms": 6.154, "p95_ms": 4.004}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.954, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1954}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4232.906, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x559fdcc356c0] error while decoding MB 16 16, bytestream -5 [h264 @ 0x559fdcc356c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559fdcc356c0] error while decoding MB 0 5, bytestream 8497 [h264 @ 0x559fdcc49840] error while decoding MB 8 23, bytestream -5 [h264 @ 0x559fdd8fec80] error while decoding MB 55 5, bytestream -7 [in#0/rtsp @ 0x559fdcc1ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559fdcc1ae00] Error retrieving a packet from demuxer: Connec", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 185, "h264_decode_error_count": 287, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T01:12:54+09:00", "last_error_summary": "[h264 @ 0x559fdcc356c0] error while decoding MB 16 16, bytestream -5 [h264 @ 0x559fdcc356c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559fdcc356c0] error while decoding MB 0 5, bytestream 8497 [h264 @ 0x559fdcc49840] error while decoding MB 8 23, bytestream -5 [h264 @ 0x559fdd8fec80] error while decoding MB 55 5, bytestream -7 [in#0/rtsp @ 0x559fdcc1ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559fdcc1ae00] Error retrieving a packet from demuxer: Connec", "last_exit_at": "2026-07-02T01:00:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 750.25, "last_started_at": "2026-07-02T01:00:24+09:00", "pid": 31797, "process": {"exists": false, "pid": 31797}, "restart_count": 210, "restart_count_per_hour": 10, "rtsp_timeout_count": 58, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 211}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.584, "sample_count": 2557, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 234528, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T01:12:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.071, "last_frame_pushed_at": "2026-07-02T01:12:54+09:00", "last_frame_read_age_sec": 0.059, "last_frame_read_at": "2026-07-02T01:12:54+09:00", "mjpeg": {"age_sec": 0.056, "exists": true, "mtime": "2026-07-02T01:12:54.419961+09:00", "mtime_epoch": 1782922374.419961, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 129222}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T01:12:54+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 978.602, "name": "celery", "pid": 237, "rss_mb": 968.102, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 103, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.44, "pushed": 3.44, "read": 3.44}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.027, "count": 103, "max_ms": 6.407, "p95_ms": 5.408}, "gstreamer_push": {"avg_ms": 1.391, "count": 103, "max_ms": 5.95, "p95_ms": 1.671}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.044, "count": 103, "max_ms": 82.077, "p95_ms": 67.991}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.363, "count": 103, "max_ms": 85.818, "p95_ms": 85.166}, "yolo11_640_fire_11class": {"avg_ms": 61.17, "count": 103, "max_ms": 82.695, "p95_ms": 76.281}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.823, "count": 103, "max_ms": 87.955, "p95_ms": 86.513}}, "truncated_count": 0}, "processing_total": {"avg_ms": 106.859, "count": 103, "max_ms": 162.471, "p95_ms": 157.59}, "read_interval": {"avg_ms": 291.1, "count": 103, "max_ms": 9412.141, "p95_ms": 171.554}, "triton_wait": {"avg_ms": 69.159, "count": 103, "max_ms": 87.972, "p95_ms": 86.525}, "visualize": {"avg_ms": 4.907, "count": 103, "max_ms": 8.856, "p95_ms": 6.714}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.941, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1954}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4232.734, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T01:12:54+09:00", "last_error_summary": "[h264 @ 0x559fdcc356c0] error while decoding MB 16 16, bytestream -5 [h264 @ 0x559fdcc356c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559fdcc356c0] error while decoding MB 0 5, bytestream 8497 [h264 @ 0x559fdcc49840] error while decoding MB 8 23, bytestream -5 [h264 @ 0x559fdd8fec80] error while decoding MB 55 5, bytestream -7 [in#0/rtsp @ 0x559fdcc1ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559fdcc1ae00] Error retrieving a packet from demuxer: Connec", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 185, "h264_decode_error_count": 287, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T01:12:54+09:00", "last_error_summary": "[h264 @ 0x559fdcc356c0] error while decoding MB 16 16, bytestream -5 [h264 @ 0x559fdcc356c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559fdcc356c0] error while decoding MB 0 5, bytestream 8497 [h264 @ 0x559fdcc49840] error while decoding MB 8 23, bytestream -5 [h264 @ 0x559fdd8fec80] error while decoding MB 55 5, bytestream -7 [in#0/rtsp @ 0x559fdcc1ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559fdcc1ae00] Error retrieving a packet from demuxer: Connec", "last_exit_at": "2026-07-02T01:12:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 750.281, "last_started_at": "2026-07-02T01:00:24+09:00", "pid": 31797, "process": {"exists": false, "pid": 31797}, "restart_count": 210, "restart_count_per_hour": 10, "rtsp_timeout_count": 58, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 211}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.584, "sample_count": 2558, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 234529, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T01:12:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T01:12:54+09:00", "last_frame_read_age_sec": 0.089, "last_frame_read_at": "2026-07-02T01:12:54+09:00", "mjpeg": {"age_sec": 0.086, "exists": true, "mtime": "2026-07-02T01:12:54.419961+09:00", "mtime_epoch": 1782922374.419961, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 129222}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T01:12:54+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 978.602, "name": "celery", "pid": 237, "rss_mb": 968.102, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 104, "pushed_frames": 104, "read_frames": 103, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.47, "pushed": 3.47, "read": 3.437}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.036, "count": 104, "max_ms": 6.407, "p95_ms": 5.408}, "gstreamer_push": {"avg_ms": 1.39, "count": 104, "max_ms": 5.95, "p95_ms": 1.671}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.162, "count": 104, "max_ms": 82.077, "p95_ms": 67.991}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.375, "count": 104, "max_ms": 85.818, "p95_ms": 85.166}, "yolo11_640_fire_11class": {"avg_ms": 61.191, "count": 104, "max_ms": 82.695, "p95_ms": 76.281}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.811, "count": 104, "max_ms": 87.955, "p95_ms": 86.513}}, "truncated_count": 0}, "processing_total": {"avg_ms": 106.608, "count": 104, "max_ms": 162.471, "p95_ms": 157.59}, "read_interval": {"avg_ms": 291.1, "count": 103, "max_ms": 9412.141, "p95_ms": 171.554}, "triton_wait": {"avg_ms": 69.145, "count": 104, "max_ms": 87.972, "p95_ms": 86.525}, "visualize": {"avg_ms": 4.898, "count": 104, "max_ms": 8.856, "p95_ms": 6.714}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.971, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1954}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4232.547, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 185, "h264_decode_error_count": 287, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T01:12:54+09:00", "last_error_summary": "[h264 @ 0x559fdcc356c0] error while decoding MB 16 16, bytestream -5 [h264 @ 0x559fdcc356c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559fdcc356c0] error while decoding MB 0 5, bytestream 8497 [h264 @ 0x559fdcc49840] error while decoding MB 8 23, bytestream -5 [h264 @ 0x559fdd8fec80] error while decoding MB 55 5, bytestream -7 [in#0/rtsp @ 0x559fdcc1ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559fdcc1ae00] Error retrieving a packet from demuxer: Connec", "last_exit_at": "2026-07-02T01:12:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 750.288, "last_started_at": "2026-07-02T01:00:24+09:00", "pid": 31797, "process": {"exists": false, "pid": 31797}, "restart_count": 210, "restart_count_per_hour": 10, "rtsp_timeout_count": 58, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 211}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.584, "sample_count": 2557, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 234529, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T01:12:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-02T01:12:54+09:00", "last_frame_read_age_sec": 0.097, "last_frame_read_at": "2026-07-02T01:12:54+09:00", "mjpeg": {"age_sec": 0.094, "exists": true, "mtime": "2026-07-02T01:12:54.419961+09:00", "mtime_epoch": 1782922374.419961, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 129222}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T01:12:54+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 978.602, "name": "celery", "pid": 237, "rss_mb": 968.102, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 104, "pushed_frames": 104, "read_frames": 103, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.469, "pushed": 3.469, "read": 3.436}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.036, "count": 104, "max_ms": 6.407, "p95_ms": 5.408}, "gstreamer_push": {"avg_ms": 1.39, "count": 104, "max_ms": 5.95, "p95_ms": 1.671}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.162, "count": 104, "max_ms": 82.077, "p95_ms": 67.991}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.375, "count": 104, "max_ms": 85.818, "p95_ms": 85.166}, "yolo11_640_fire_11class": {"avg_ms": 61.191, "count": 104, "max_ms": 82.695, "p95_ms": 76.281}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.811, "count": 104, "max_ms": 87.955, "p95_ms": 86.513}}, "truncated_count": 0}, "processing_total": {"avg_ms": 106.608, "count": 104, "max_ms": 162.471, "p95_ms": 157.59}, "read_interval": {"avg_ms": 291.1, "count": 103, "max_ms": 9412.141, "p95_ms": 171.554}, "triton_wait": {"avg_ms": 69.145, "count": 104, "max_ms": 87.972, "p95_ms": 86.525}, "visualize": {"avg_ms": 4.898, "count": 104, "max_ms": 8.856, "p95_ms": 6.714}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.979, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 1954}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4345.426, "limit_mb": 8796093022207.996}, "detail": {"pid": 31904, "restart_count": 211, "start_count": 212}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 185, "h264_decode_error_count": 287, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T01:12:54+09:00", "last_error_summary": "[h264 @ 0x559fdcc356c0] error while decoding MB 16 16, bytestream -5 [h264 @ 0x559fdcc356c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559fdcc356c0] error while decoding MB 0 5, bytestream 8497 [h264 @ 0x559fdcc49840] error while decoding MB 8 23, bytestream -5 [h264 @ 0x559fdd8fec80] error while decoding MB 55 5, bytestream -7 [in#0/rtsp @ 0x559fdcc1ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559fdcc1ae00] Error retrieving a packet from demuxer: Connec", "last_exit_at": "2026-07-02T01:12:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T01:12:56+09:00", "pid": 31904, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.512, "name": "ffmpeg", "pid": 31904, "rss_mb": 141.512, "state": "S (sleeping)", "threads": 106}, "restart_count": 211, "restart_count_per_hour": 11, "rtsp_timeout_count": 58, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 212}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.631, "sample_count": 2518, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 234529, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T01:12:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.137, "last_frame_pushed_at": "2026-07-02T01:12:54+09:00", "last_frame_read_age_sec": 4.22, "last_frame_read_at": "2026-07-02T01:12:54+09:00", "mjpeg": {"age_sec": 4.125, "exists": true, "mtime": "2026-07-02T01:12:54.511965+09:00", "mtime_epoch": 1782922374.5119646, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 131005}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T01:12:58+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 978.602, "name": "celery", "pid": 237, "rss_mb": 968.102, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 72, "pushed_frames": 72, "read_frames": 71, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.402, "pushed": 2.402, "read": 2.368}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.992, "count": 71, "max_ms": 5.519, "p95_ms": 5.263}, "gstreamer_push": {"avg_ms": 1.351, "count": 72, "max_ms": 5.95, "p95_ms": 1.576}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.903, "count": 71, "max_ms": 82.077, "p95_ms": 67.933}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.02, "count": 71, "max_ms": 84.122, "p95_ms": 80.305}, "yolo11_640_fire_11class": {"avg_ms": 59.333, "count": 71, "max_ms": 82.695, "p95_ms": 76.281}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.939, "count": 71, "max_ms": 87.852, "p95_ms": 85.045}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.072, "count": 72, "max_ms": 162.209, "p95_ms": 158.764}, "read_interval": {"avg_ms": 364.756, "count": 71, "max_ms": 9412.141, "p95_ms": 172.137}, "triton_wait": {"avg_ms": 66.345, "count": 71, "max_ms": 87.865, "p95_ms": 85.057}, "visualize": {"avg_ms": 4.797, "count": 71, "max_ms": 6.997, "p95_ms": 6.788}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.979, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2028}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4251.395, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 8, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5644498fc280] top block unavailable for requested intra mode -1 [h264 @ 0x5644498fc280] error while decoding MB 23 0, bytestream 4559 [h264 @ 0x56444a568d80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56444a568d80] error while decoding MB 0 2, bytestream 7683 [h264 @ 0x5644498a65c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5644498a65c0] error while decoding MB 0 30, bytestream 694 [h264 @ 0x564449998f40] top block unavailable for requested in", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 186, "h264_decode_error_count": 295, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T01:28:53+09:00", "last_error_summary": "[h264 @ 0x5644498fc280] top block unavailable for requested intra mode -1 [h264 @ 0x5644498fc280] error while decoding MB 23 0, bytestream 4559 [h264 @ 0x56444a568d80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56444a568d80] error while decoding MB 0 2, bytestream 7683 [h264 @ 0x5644498a65c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5644498a65c0] error while decoding MB 0 30, bytestream 694 [h264 @ 0x564449998f40] top block unavailable for requested in", "last_exit_at": "2026-07-02T01:12:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 956.998, "last_started_at": "2026-07-02T01:12:56+09:00", "pid": 31904, "process": {"exists": false, "pid": 31904}, "restart_count": 211, "restart_count_per_hour": 8, "rtsp_timeout_count": 60, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 212}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 134.295, "sample_count": 2626, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 243467, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T01:28:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.07, "last_frame_pushed_at": "2026-07-02T01:28:53+09:00", "last_frame_read_age_sec": 0.057, "last_frame_read_at": "2026-07-02T01:28:53+09:00", "mjpeg": {"age_sec": 0.053, "exists": true, "mtime": "2026-07-02T01:28:53.581795+09:00", "mtime_epoch": 1782923333.581795, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 145176}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T01:28:53+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 983.336, "name": "celery", "pid": 237, "rss_mb": 980.074, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 100, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.335, "pushed": 3.335, "read": 3.335}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.539, "count": 100, "max_ms": 2.079, "p95_ms": 1.588}, "gstreamer_push": {"avg_ms": 1.64, "count": 100, "max_ms": 6.197, "p95_ms": 3.943}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.439, "count": 100, "max_ms": 78.434, "p95_ms": 76.315}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.734, "count": 100, "max_ms": 80.775, "p95_ms": 79.024}, "yolo11_640_fire_11class": {"avg_ms": 72.978, "count": 100, "max_ms": 79.584, "p95_ms": 77.217}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.686, "count": 100, "max_ms": 89.625, "p95_ms": 87.348}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.229, "count": 100, "max_ms": 99.99, "p95_ms": 97.941}, "read_interval": {"avg_ms": 300.166, "count": 100, "max_ms": 19148.327, "p95_ms": 115.496}, "triton_wait": {"avg_ms": 82.782, "count": 100, "max_ms": 89.637, "p95_ms": 87.36}, "visualize": {"avg_ms": 3.719, "count": 100, "max_ms": 4.613, "p95_ms": 3.933}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.981, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2028}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4250.766, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T01:28:53+09:00", "last_error_summary": "[h264 @ 0x5644498fc280] top block unavailable for requested intra mode -1 [h264 @ 0x5644498fc280] error while decoding MB 23 0, bytestream 4559 [h264 @ 0x56444a568d80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56444a568d80] error while decoding MB 0 2, bytestream 7683 [h264 @ 0x5644498a65c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5644498a65c0] error while decoding MB 0 30, bytestream 694 [h264 @ 0x564449998f40] top block unavailable for requested in", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 186, "h264_decode_error_count": 295, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T01:28:53+09:00", "last_error_summary": "[h264 @ 0x5644498fc280] top block unavailable for requested intra mode -1 [h264 @ 0x5644498fc280] error while decoding MB 23 0, bytestream 4559 [h264 @ 0x56444a568d80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56444a568d80] error while decoding MB 0 2, bytestream 7683 [h264 @ 0x5644498a65c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5644498a65c0] error while decoding MB 0 30, bytestream 694 [h264 @ 0x564449998f40] top block unavailable for requested in", "last_exit_at": "2026-07-02T01:28:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 957.048, "last_started_at": "2026-07-02T01:12:56+09:00", "pid": 31904, "process": {"exists": false, "pid": 31904}, "restart_count": 211, "restart_count_per_hour": 8, "rtsp_timeout_count": 60, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 212}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 134.295, "sample_count": 2627, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 243468, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T01:28:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-02T01:28:53+09:00", "last_frame_read_age_sec": 0.107, "last_frame_read_at": "2026-07-02T01:28:53+09:00", "mjpeg": {"age_sec": 0.102, "exists": true, "mtime": "2026-07-02T01:28:53.581795+09:00", "mtime_epoch": 1782923333.581795, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 145176}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T01:28:53+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 983.336, "name": "celery", "pid": 237, "rss_mb": 980.074, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.342, "pushed": 3.342, "read": 3.309}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.539, "count": 100, "max_ms": 2.079, "p95_ms": 1.588}, "gstreamer_push": {"avg_ms": 1.642, "count": 100, "max_ms": 6.197, "p95_ms": 3.943}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.299, "count": 100, "max_ms": 78.434, "p95_ms": 76.315}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.741, "count": 100, "max_ms": 80.775, "p95_ms": 79.024}, "yolo11_640_fire_11class": {"avg_ms": 73.009, "count": 100, "max_ms": 79.584, "p95_ms": 77.217}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.703, "count": 100, "max_ms": 89.625, "p95_ms": 87.348}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.25, "count": 100, "max_ms": 99.99, "p95_ms": 97.941}, "read_interval": {"avg_ms": 302.098, "count": 99, "max_ms": 19148.327, "p95_ms": 115.687}, "triton_wait": {"avg_ms": 82.799, "count": 100, "max_ms": 89.637, "p95_ms": 87.36}, "visualize": {"avg_ms": 3.719, "count": 100, "max_ms": 4.613, "p95_ms": 3.933}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.923, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2028}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4250.734, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 186, "h264_decode_error_count": 295, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T01:28:53+09:00", "last_error_summary": "[h264 @ 0x5644498fc280] top block unavailable for requested intra mode -1 [h264 @ 0x5644498fc280] error while decoding MB 23 0, bytestream 4559 [h264 @ 0x56444a568d80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56444a568d80] error while decoding MB 0 2, bytestream 7683 [h264 @ 0x5644498a65c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5644498a65c0] error while decoding MB 0 30, bytestream 694 [h264 @ 0x564449998f40] top block unavailable for requested in", "last_exit_at": "2026-07-02T01:28:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 957.056, "last_started_at": "2026-07-02T01:12:56+09:00", "pid": 31904, "process": {"exists": false, "pid": 31904}, "restart_count": 211, "restart_count_per_hour": 8, "rtsp_timeout_count": 60, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 212}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 134.295, "sample_count": 2627, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 243468, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T01:28:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.017, "last_frame_pushed_at": "2026-07-02T01:28:53+09:00", "last_frame_read_age_sec": 0.115, "last_frame_read_at": "2026-07-02T01:28:53+09:00", "mjpeg": {"age_sec": 0.11, "exists": true, "mtime": "2026-07-02T01:28:53.581795+09:00", "mtime_epoch": 1782923333.581795, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 145176}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T01:28:53+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 983.336, "name": "celery", "pid": 237, "rss_mb": 980.074, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.341, "pushed": 3.341, "read": 3.308}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.539, "count": 100, "max_ms": 2.079, "p95_ms": 1.588}, "gstreamer_push": {"avg_ms": 1.642, "count": 100, "max_ms": 6.197, "p95_ms": 3.943}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.299, "count": 100, "max_ms": 78.434, "p95_ms": 76.315}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.741, "count": 100, "max_ms": 80.775, "p95_ms": 79.024}, "yolo11_640_fire_11class": {"avg_ms": 73.009, "count": 100, "max_ms": 79.584, "p95_ms": 77.217}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.703, "count": 100, "max_ms": 89.625, "p95_ms": 87.348}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.25, "count": 100, "max_ms": 99.99, "p95_ms": 97.941}, "read_interval": {"avg_ms": 302.098, "count": 99, "max_ms": 19148.327, "p95_ms": 115.687}, "triton_wait": {"avg_ms": 82.799, "count": 100, "max_ms": 89.637, "p95_ms": 87.36}, "visualize": {"avg_ms": 3.719, "count": 100, "max_ms": 4.613, "p95_ms": 3.933}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.93, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2028}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4352.453, "limit_mb": 8796093022207.996}, "detail": {"pid": 32011, "restart_count": 212, "start_count": 213}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 186, "h264_decode_error_count": 295, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T01:28:53+09:00", "last_error_summary": "[h264 @ 0x5644498fc280] top block unavailable for requested intra mode -1 [h264 @ 0x5644498fc280] error while decoding MB 23 0, bytestream 4559 [h264 @ 0x56444a568d80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56444a568d80] error while decoding MB 0 2, bytestream 7683 [h264 @ 0x5644498a65c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5644498a65c0] error while decoding MB 0 30, bytestream 694 [h264 @ 0x564449998f40] top block unavailable for requested in", "last_exit_at": "2026-07-02T01:28:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T01:28:56+09:00", "pid": 32011, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.078, "name": "ffmpeg", "pid": 32011, "rss_mb": 139.078, "state": "S (sleeping)", "threads": 106}, "restart_count": 212, "restart_count_per_hour": 8, "rtsp_timeout_count": 60, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 213}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 134.344, "sample_count": 2581, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 243468, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T01:28:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.672, "last_frame_pushed_at": "2026-07-02T01:28:53+09:00", "last_frame_read_age_sec": 4.769, "last_frame_read_at": "2026-07-02T01:28:53+09:00", "mjpeg": {"age_sec": 4.657, "exists": true, "mtime": "2026-07-02T01:28:53.689797+09:00", "mtime_epoch": 1782923333.6897974, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 145805}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T01:28:58+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 983.336, "name": "celery", "pid": 237, "rss_mb": 980.074, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 58, "pushed_frames": 58, "read_frames": 57, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.937, "pushed": 1.937, "read": 1.903}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.542, "count": 58, "max_ms": 2.079, "p95_ms": 1.647}, "gstreamer_push": {"avg_ms": 1.531, "count": 58, "max_ms": 3.957, "p95_ms": 3.663}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.983, "count": 58, "max_ms": 78.434, "p95_ms": 77.108}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.19, "count": 58, "max_ms": 80.775, "p95_ms": 79.593}, "yolo11_640_fire_11class": {"avg_ms": 72.266, "count": 58, "max_ms": 78.972, "p95_ms": 77.331}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.607, "count": 58, "max_ms": 89.625, "p95_ms": 87.597}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.194, "count": 58, "max_ms": 99.99, "p95_ms": 98.028}, "read_interval": {"avg_ms": 443.452, "count": 57, "max_ms": 19148.327, "p95_ms": 115.496}, "triton_wait": {"avg_ms": 82.745, "count": 58, "max_ms": 89.637, "p95_ms": 87.609}, "visualize": {"avg_ms": 3.73, "count": 58, "max_ms": 4.613, "p95_ms": 4.176}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.95, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2046}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4245.363, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 5, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x559fc5ff0f80] cabac decode of qscale diff failed at 32 19 [h264 @ 0x559fc5ff0f80] error while decoding MB 32 19, bytestream 41821 [h264 @ 0x559fc5775a80] cabac decode of qscale diff failed at 14 18 [h264 @ 0x559fc5775a80] error while decoding MB 14 18, bytestream 9874 [h264 @ 0x559fc60bfc80] error while decoding MB 5 7, bytestream -14 [in#0/rtsp @ 0x559fc5641e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559fc5641e00] Error retrieving a packet from demuxer: Connection ", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 187, "h264_decode_error_count": 300, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T01:33:18+09:00", "last_error_summary": "[h264 @ 0x559fc5ff0f80] cabac decode of qscale diff failed at 32 19 [h264 @ 0x559fc5ff0f80] error while decoding MB 32 19, bytestream 41821 [h264 @ 0x559fc5775a80] cabac decode of qscale diff failed at 14 18 [h264 @ 0x559fc5775a80] error while decoding MB 14 18, bytestream 9874 [h264 @ 0x559fc60bfc80] error while decoding MB 5 7, bytestream -14 [in#0/rtsp @ 0x559fc5641e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559fc5641e00] Error retrieving a packet from demuxer: Connection ", "last_exit_at": "2026-07-02T01:28:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 262.393, "last_started_at": "2026-07-02T01:28:56+09:00", "pid": 32011, "process": {"exists": false, "pid": 32011}, "restart_count": 212, "restart_count_per_hour": 8, "rtsp_timeout_count": 62, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 213}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.598, "sample_count": 2267, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 245590, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T01:33:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.074, "last_frame_pushed_at": "2026-07-02T01:33:18+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-02T01:33:18+09:00", "mjpeg": {"age_sec": 0.055, "exists": true, "mtime": "2026-07-02T01:33:18.683270+09:00", "mtime_epoch": 1782923598.6832702, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 117708}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T01:33:18+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 983.336, "name": "celery", "pid": 237, "rss_mb": 975.488, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 102, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.402, "pushed": 3.402, "read": 3.402}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.319, "count": 101, "max_ms": 5.785, "p95_ms": 4.025}, "gstreamer_push": {"avg_ms": 1.561, "count": 102, "max_ms": 6.048, "p95_ms": 3.814}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.455, "count": 101, "max_ms": 75.826, "p95_ms": 62.666}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.629, "count": 101, "max_ms": 87.121, "p95_ms": 84.042}, "yolo11_640_fire_11class": {"avg_ms": 56.318, "count": 101, "max_ms": 75.389, "p95_ms": 73.187}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.111, "count": 101, "max_ms": 93.475, "p95_ms": 85.655}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.798, "count": 102, "max_ms": 144.618, "p95_ms": 135.552}, "read_interval": {"avg_ms": 294.398, "count": 102, "max_ms": 10174.893, "p95_ms": 152.796}, "triton_wait": {"avg_ms": 63.113, "count": 101, "max_ms": 93.488, "p95_ms": 86.334}, "visualize": {"avg_ms": 4.168, "count": 101, "max_ms": 9.133, "p95_ms": 5.244}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.981, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2046}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4245.004, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T01:33:18+09:00", "last_error_summary": "[h264 @ 0x559fc5ff0f80] cabac decode of qscale diff failed at 32 19 [h264 @ 0x559fc5ff0f80] error while decoding MB 32 19, bytestream 41821 [h264 @ 0x559fc5775a80] cabac decode of qscale diff failed at 14 18 [h264 @ 0x559fc5775a80] error while decoding MB 14 18, bytestream 9874 [h264 @ 0x559fc60bfc80] error while decoding MB 5 7, bytestream -14 [in#0/rtsp @ 0x559fc5641e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559fc5641e00] Error retrieving a packet from demuxer: Connection ", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 187, "h264_decode_error_count": 300, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T01:33:18+09:00", "last_error_summary": "[h264 @ 0x559fc5ff0f80] cabac decode of qscale diff failed at 32 19 [h264 @ 0x559fc5ff0f80] error while decoding MB 32 19, bytestream 41821 [h264 @ 0x559fc5775a80] cabac decode of qscale diff failed at 14 18 [h264 @ 0x559fc5775a80] error while decoding MB 14 18, bytestream 9874 [h264 @ 0x559fc60bfc80] error while decoding MB 5 7, bytestream -14 [in#0/rtsp @ 0x559fc5641e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559fc5641e00] Error retrieving a packet from demuxer: Connection ", "last_exit_at": "2026-07-02T01:33:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 262.436, "last_started_at": "2026-07-02T01:28:56+09:00", "pid": 32011, "process": {"exists": false, "pid": 32011}, "restart_count": 212, "restart_count_per_hour": 8, "rtsp_timeout_count": 62, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 213}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.598, "sample_count": 2268, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 245591, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T01:33:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T01:33:18+09:00", "last_frame_read_age_sec": 0.104, "last_frame_read_at": "2026-07-02T01:33:18+09:00", "mjpeg": {"age_sec": 0.098, "exists": true, "mtime": "2026-07-02T01:33:18.683270+09:00", "mtime_epoch": 1782923598.6832702, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 117708}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T01:33:18+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 983.336, "name": "celery", "pid": 237, "rss_mb": 975.488, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 101, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.414, "pushed": 3.414, "read": 3.38}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.312, "count": 102, "max_ms": 5.785, "p95_ms": 4.025}, "gstreamer_push": {"avg_ms": 1.563, "count": 102, "max_ms": 6.048, "p95_ms": 3.814}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.55, "count": 102, "max_ms": 75.826, "p95_ms": 62.666}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.795, "count": 102, "max_ms": 87.121, "p95_ms": 84.042}, "yolo11_640_fire_11class": {"avg_ms": 56.452, "count": 102, "max_ms": 75.389, "p95_ms": 73.187}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.338, "count": 102, "max_ms": 93.475, "p95_ms": 85.655}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.67, "count": 102, "max_ms": 144.618, "p95_ms": 135.552}, "read_interval": {"avg_ms": 296.108, "count": 101, "max_ms": 10174.893, "p95_ms": 152.796}, "triton_wait": {"avg_ms": 63.33, "count": 102, "max_ms": 93.488, "p95_ms": 86.334}, "visualize": {"avg_ms": 4.163, "count": 102, "max_ms": 9.133, "p95_ms": 5.244}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.879, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2046}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4244.844, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 187, "h264_decode_error_count": 300, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T01:33:18+09:00", "last_error_summary": "[h264 @ 0x559fc5ff0f80] cabac decode of qscale diff failed at 32 19 [h264 @ 0x559fc5ff0f80] error while decoding MB 32 19, bytestream 41821 [h264 @ 0x559fc5775a80] cabac decode of qscale diff failed at 14 18 [h264 @ 0x559fc5775a80] error while decoding MB 14 18, bytestream 9874 [h264 @ 0x559fc60bfc80] error while decoding MB 5 7, bytestream -14 [in#0/rtsp @ 0x559fc5641e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559fc5641e00] Error retrieving a packet from demuxer: Connection ", "last_exit_at": "2026-07-02T01:33:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 262.445, "last_started_at": "2026-07-02T01:28:56+09:00", "pid": 32011, "process": {"exists": false, "pid": 32011}, "restart_count": 212, "restart_count_per_hour": 8, "rtsp_timeout_count": 62, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 213}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.598, "sample_count": 2267, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 245591, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T01:33:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-02T01:33:18+09:00", "last_frame_read_age_sec": 0.114, "last_frame_read_at": "2026-07-02T01:33:18+09:00", "mjpeg": {"age_sec": 0.107, "exists": true, "mtime": "2026-07-02T01:33:18.683270+09:00", "mtime_epoch": 1782923598.6832702, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 117708}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T01:33:18+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 983.336, "name": "celery", "pid": 237, "rss_mb": 975.488, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 101, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.413, "pushed": 3.413, "read": 3.379}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.312, "count": 102, "max_ms": 5.785, "p95_ms": 4.025}, "gstreamer_push": {"avg_ms": 1.563, "count": 102, "max_ms": 6.048, "p95_ms": 3.814}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.55, "count": 102, "max_ms": 75.826, "p95_ms": 62.666}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.795, "count": 102, "max_ms": 87.121, "p95_ms": 84.042}, "yolo11_640_fire_11class": {"avg_ms": 56.452, "count": 102, "max_ms": 75.389, "p95_ms": 73.187}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.338, "count": 102, "max_ms": 93.475, "p95_ms": 85.655}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.67, "count": 102, "max_ms": 144.618, "p95_ms": 135.552}, "read_interval": {"avg_ms": 296.108, "count": 101, "max_ms": 10174.893, "p95_ms": 152.796}, "triton_wait": {"avg_ms": 63.33, "count": 102, "max_ms": 93.488, "p95_ms": 86.334}, "visualize": {"avg_ms": 4.163, "count": 102, "max_ms": 9.133, "p95_ms": 5.244}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.889, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2046}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4352.832, "limit_mb": 8796093022207.996}, "detail": {"pid": 32118, "restart_count": 213, "start_count": 214}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 187, "h264_decode_error_count": 300, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T01:33:18+09:00", "last_error_summary": "[h264 @ 0x559fc5ff0f80] cabac decode of qscale diff failed at 32 19 [h264 @ 0x559fc5ff0f80] error while decoding MB 32 19, bytestream 41821 [h264 @ 0x559fc5775a80] cabac decode of qscale diff failed at 14 18 [h264 @ 0x559fc5775a80] error while decoding MB 14 18, bytestream 9874 [h264 @ 0x559fc60bfc80] error while decoding MB 5 7, bytestream -14 [in#0/rtsp @ 0x559fc5641e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559fc5641e00] Error retrieving a packet from demuxer: Connection ", "last_exit_at": "2026-07-02T01:33:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T01:33:21+09:00", "pid": 32118, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.02, "name": "ffmpeg", "pid": 32118, "rss_mb": 139.02, "state": "S (sleeping)", "threads": 106}, "restart_count": 213, "restart_count_per_hour": 9, "rtsp_timeout_count": 62, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 214}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.668, "sample_count": 2226, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 245591, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T01:33:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.564, "last_frame_pushed_at": "2026-07-02T01:33:18+09:00", "last_frame_read_age_sec": 4.662, "last_frame_read_at": "2026-07-02T01:33:18+09:00", "mjpeg": {"age_sec": 4.548, "exists": true, "mtime": "2026-07-02T01:33:18.791272+09:00", "mtime_epoch": 1782923598.7912724, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 116985}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T01:33:23+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 983.336, "name": "celery", "pid": 237, "rss_mb": 975.488, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 62, "pushed_frames": 62, "read_frames": 61, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.07, "pushed": 2.07, "read": 2.037}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.773, "count": 62, "max_ms": 4.025, "p95_ms": 3.916}, "gstreamer_push": {"avg_ms": 1.739, "count": 62, "max_ms": 6.048, "p95_ms": 3.865}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.17, "count": 62, "max_ms": 72.643, "p95_ms": 58.184}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.143, "count": 62, "max_ms": 87.121, "p95_ms": 85.773}, "yolo11_640_fire_11class": {"avg_ms": 56.491, "count": 62, "max_ms": 74.884, "p95_ms": 73.187}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.967, "count": 62, "max_ms": 86.892, "p95_ms": 85.23}}, "truncated_count": 0}, "processing_total": {"avg_ms": 76.892, "count": 62, "max_ms": 136.529, "p95_ms": 106.419}, "read_interval": {"avg_ms": 415.613, "count": 61, "max_ms": 10174.893, "p95_ms": 132.21}, "triton_wait": {"avg_ms": 63.117, "count": 62, "max_ms": 87.133, "p95_ms": 86.19}, "visualize": {"avg_ms": 3.791, "count": 62, "max_ms": 5.276, "p95_ms": 4.959}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.947, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2059}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4251.227, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 6, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x559f8e5c9e00] error while decoding MB 77 34, bytestream -7 [h264 @ 0x559f8da3a140] cabac decode of qscale diff failed at 70 13 [h264 @ 0x559f8da3a140] error while decoding MB 70 13, bytestream 5816 [h264 @ 0x559f8e4faf80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559f8e4faf80] error while decoding MB 0 5, bytestream 45343 [h264 @ 0x559f8e35d340] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559f8e35d340] error while decoding MB 0 33, bytestream 7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 188, "h264_decode_error_count": 306, "h264_decode_error_count_per_hour": 53, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T01:36:40+09:00", "last_error_summary": "[h264 @ 0x559f8e5c9e00] error while decoding MB 77 34, bytestream -7 [h264 @ 0x559f8da3a140] cabac decode of qscale diff failed at 70 13 [h264 @ 0x559f8da3a140] error while decoding MB 70 13, bytestream 5816 [h264 @ 0x559f8e4faf80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559f8e4faf80] error while decoding MB 0 5, bytestream 45343 [h264 @ 0x559f8e35d340] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559f8e35d340] error while decoding MB 0 33, bytestream 7", "last_exit_at": "2026-07-02T01:33:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 199.414, "last_started_at": "2026-07-02T01:33:21+09:00", "pid": 32118, "process": {"exists": false, "pid": 32118}, "restart_count": 213, "restart_count_per_hour": 8, "rtsp_timeout_count": 64, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 214}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.516, "sample_count": 2266, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 247156, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T01:36:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-02T01:36:40+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-02T01:36:40+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-02T01:36:40.707274+09:00", "mtime_epoch": 1782923800.7072737, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 117994}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T01:36:40+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 983.336, "name": "celery", "pid": 237, "rss_mb": 979.938, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 94, "pushed_frames": 94, "read_frames": 94, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.144, "pushed": 3.144, "read": 3.144}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.649, "count": 94, "max_ms": 5.104, "p95_ms": 4.375}, "gstreamer_push": {"avg_ms": 1.533, "count": 94, "max_ms": 5.861, "p95_ms": 3.829}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.954, "count": 94, "max_ms": 76.847, "p95_ms": 69.602}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.485, "count": 94, "max_ms": 87.428, "p95_ms": 85.846}, "yolo11_640_fire_11class": {"avg_ms": 64.158, "count": 94, "max_ms": 77.375, "p95_ms": 75.389}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.041, "count": 94, "max_ms": 88.659, "p95_ms": 86.817}}, "truncated_count": 0}, "processing_total": {"avg_ms": 101.742, "count": 94, "max_ms": 171.335, "p95_ms": 137.576}, "read_interval": {"avg_ms": 318.952, "count": 94, "max_ms": 10129.513, "p95_ms": 153.778}, "triton_wait": {"avg_ms": 73.945, "count": 94, "max_ms": 88.675, "p95_ms": 87.012}, "visualize": {"avg_ms": 4.362, "count": 94, "max_ms": 6.209, "p95_ms": 5.245}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.899, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2059}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4250.324, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T01:36:40+09:00", "last_error_summary": "[h264 @ 0x559f8e5c9e00] error while decoding MB 77 34, bytestream -7 [h264 @ 0x559f8da3a140] cabac decode of qscale diff failed at 70 13 [h264 @ 0x559f8da3a140] error while decoding MB 70 13, bytestream 5816 [h264 @ 0x559f8e4faf80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559f8e4faf80] error while decoding MB 0 5, bytestream 45343 [h264 @ 0x559f8e35d340] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559f8e35d340] error while decoding MB 0 33, bytestream 7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 188, "h264_decode_error_count": 306, "h264_decode_error_count_per_hour": 53, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T01:36:40+09:00", "last_error_summary": "[h264 @ 0x559f8e5c9e00] error while decoding MB 77 34, bytestream -7 [h264 @ 0x559f8da3a140] cabac decode of qscale diff failed at 70 13 [h264 @ 0x559f8da3a140] error while decoding MB 70 13, bytestream 5816 [h264 @ 0x559f8e4faf80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559f8e4faf80] error while decoding MB 0 5, bytestream 45343 [h264 @ 0x559f8e35d340] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559f8e35d340] error while decoding MB 0 33, bytestream 7", "last_exit_at": "2026-07-02T01:36:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 199.511, "last_started_at": "2026-07-02T01:33:21+09:00", "pid": 32118, "process": {"exists": false, "pid": 32118}, "restart_count": 213, "restart_count_per_hour": 8, "rtsp_timeout_count": 64, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 214}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.52, "sample_count": 2267, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 247157, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T01:36:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T01:36:40+09:00", "last_frame_read_age_sec": 0.147, "last_frame_read_at": "2026-07-02T01:36:40+09:00", "mjpeg": {"age_sec": 0.142, "exists": true, "mtime": "2026-07-02T01:36:40.707274+09:00", "mtime_epoch": 1782923800.7072737, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 117994}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T01:36:40+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 983.336, "name": "celery", "pid": 237, "rss_mb": 979.938, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 95, "pushed_frames": 95, "read_frames": 94, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.167, "pushed": 3.167, "read": 3.134}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.648, "count": 94, "max_ms": 5.104, "p95_ms": 4.375}, "gstreamer_push": {"avg_ms": 1.529, "count": 95, "max_ms": 5.861, "p95_ms": 3.829}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.064, "count": 94, "max_ms": 76.847, "p95_ms": 71.064}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.475, "count": 94, "max_ms": 87.428, "p95_ms": 85.846}, "yolo11_640_fire_11class": {"avg_ms": 64.206, "count": 94, "max_ms": 77.375, "p95_ms": 75.389}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.08, "count": 94, "max_ms": 88.659, "p95_ms": 86.817}}, "truncated_count": 0}, "processing_total": {"avg_ms": 102.129, "count": 95, "max_ms": 171.335, "p95_ms": 137.647}, "read_interval": {"avg_ms": 318.952, "count": 94, "max_ms": 10129.513, "p95_ms": 153.778}, "triton_wait": {"avg_ms": 73.984, "count": 94, "max_ms": 88.675, "p95_ms": 87.012}, "visualize": {"avg_ms": 4.361, "count": 94, "max_ms": 6.209, "p95_ms": 5.245}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2059}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4250.312, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 188, "h264_decode_error_count": 306, "h264_decode_error_count_per_hour": 53, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T01:36:40+09:00", "last_error_summary": "[h264 @ 0x559f8e5c9e00] error while decoding MB 77 34, bytestream -7 [h264 @ 0x559f8da3a140] cabac decode of qscale diff failed at 70 13 [h264 @ 0x559f8da3a140] error while decoding MB 70 13, bytestream 5816 [h264 @ 0x559f8e4faf80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559f8e4faf80] error while decoding MB 0 5, bytestream 45343 [h264 @ 0x559f8e35d340] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559f8e35d340] error while decoding MB 0 33, bytestream 7", "last_exit_at": "2026-07-02T01:36:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 199.516, "last_started_at": "2026-07-02T01:33:21+09:00", "pid": 32118, "process": {"exists": false, "pid": 32118}, "restart_count": 213, "restart_count_per_hour": 8, "rtsp_timeout_count": 64, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 214}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.52, "sample_count": 2267, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 247157, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T01:36:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-02T01:36:40+09:00", "last_frame_read_age_sec": 0.152, "last_frame_read_at": "2026-07-02T01:36:40+09:00", "mjpeg": {"age_sec": 0.147, "exists": true, "mtime": "2026-07-02T01:36:40.707274+09:00", "mtime_epoch": 1782923800.7072737, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 117994}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T01:36:40+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 983.336, "name": "celery", "pid": 237, "rss_mb": 979.938, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 94, "pushed_frames": 95, "read_frames": 94, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.133, "pushed": 3.167, "read": 3.133}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.648, "count": 94, "max_ms": 5.104, "p95_ms": 4.375}, "gstreamer_push": {"avg_ms": 1.529, "count": 95, "max_ms": 5.861, "p95_ms": 3.829}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.064, "count": 94, "max_ms": 76.847, "p95_ms": 71.064}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.475, "count": 94, "max_ms": 87.428, "p95_ms": 85.846}, "yolo11_640_fire_11class": {"avg_ms": 64.206, "count": 94, "max_ms": 77.375, "p95_ms": 75.389}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.08, "count": 94, "max_ms": 88.659, "p95_ms": 86.817}}, "truncated_count": 0}, "processing_total": {"avg_ms": 101.804, "count": 94, "max_ms": 171.335, "p95_ms": 137.647}, "read_interval": {"avg_ms": 318.952, "count": 94, "max_ms": 10129.513, "p95_ms": 153.778}, "triton_wait": {"avg_ms": 73.984, "count": 94, "max_ms": 88.675, "p95_ms": 87.012}, "visualize": {"avg_ms": 4.361, "count": 94, "max_ms": 6.209, "p95_ms": 5.245}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2059}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4360.402, "limit_mb": 8796093022207.996}, "detail": {"pid": 32225, "restart_count": 214, "start_count": 215}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 188, "h264_decode_error_count": 306, "h264_decode_error_count_per_hour": 53, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T01:36:40+09:00", "last_error_summary": "[h264 @ 0x559f8e5c9e00] error while decoding MB 77 34, bytestream -7 [h264 @ 0x559f8da3a140] cabac decode of qscale diff failed at 70 13 [h264 @ 0x559f8da3a140] error while decoding MB 70 13, bytestream 5816 [h264 @ 0x559f8e4faf80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559f8e4faf80] error while decoding MB 0 5, bytestream 45343 [h264 @ 0x559f8e35d340] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559f8e35d340] error while decoding MB 0 33, bytestream 7", "last_exit_at": "2026-07-02T01:36:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T01:36:43+09:00", "pid": 32225, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.141, "name": "ffmpeg", "pid": 32225, "rss_mb": 139.141, "state": "S (sleeping)", "threads": 106}, "restart_count": 214, "restart_count_per_hour": 9, "rtsp_timeout_count": 64, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 215}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.52, "sample_count": 2230, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 247157, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T01:36:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.64, "last_frame_pushed_at": "2026-07-02T01:36:40+09:00", "last_frame_read_age_sec": 4.78, "last_frame_read_at": "2026-07-02T01:36:40+09:00", "mjpeg": {"age_sec": 4.623, "exists": true, "mtime": "2026-07-02T01:36:40.859277+09:00", "mtime_epoch": 1782923800.8592768, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 117911}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T01:36:45+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 983.336, "name": "celery", "pid": 237, "rss_mb": 979.938, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 56, "pushed_frames": 56, "read_frames": 55, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.868, "pushed": 1.868, "read": 1.835}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.47, "count": 56, "max_ms": 4.313, "p95_ms": 4.245}, "gstreamer_push": {"avg_ms": 1.424, "count": 56, "max_ms": 4.173, "p95_ms": 1.578}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.159, "count": 56, "max_ms": 74.899, "p95_ms": 71.064}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.036, "count": 56, "max_ms": 87.428, "p95_ms": 85.472}, "yolo11_640_fire_11class": {"avg_ms": 64.87, "count": 56, "max_ms": 76.027, "p95_ms": 75.548}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.422, "count": 56, "max_ms": 88.659, "p95_ms": 86.817}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.913, "count": 56, "max_ms": 143.169, "p95_ms": 138.511}, "read_interval": {"avg_ms": 459.901, "count": 55, "max_ms": 10129.513, "p95_ms": 155.046}, "triton_wait": {"avg_ms": 74.364, "count": 56, "max_ms": 88.675, "p95_ms": 87.441}, "visualize": {"avg_ms": 4.319, "count": 56, "max_ms": 5.99, "p95_ms": 5.213}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.979, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2069}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4254.742, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x555d4c78b580] left block unavailable for requested intra mode [h264 @ 0x555d4c78b580] error while decoding MB 0 39, bytestream 28138 [h264 @ 0x555d4c9f7f80] error while decoding MB 20 15, bytestream -13 [in#0/rtsp @ 0x555d4bde1e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555d4bde1e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 189, "h264_decode_error_count": 308, "h264_decode_error_count_per_hour": 49, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T01:39:22+09:00", "last_error_summary": "[h264 @ 0x555d4c78b580] left block unavailable for requested intra mode [h264 @ 0x555d4c78b580] error while decoding MB 0 39, bytestream 28138 [h264 @ 0x555d4c9f7f80] error while decoding MB 20 15, bytestream -13 [in#0/rtsp @ 0x555d4bde1e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555d4bde1e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T01:36:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 159.516, "last_started_at": "2026-07-02T01:36:43+09:00", "pid": 32225, "process": {"exists": false, "pid": 32225}, "restart_count": 214, "restart_count_per_hour": 8, "rtsp_timeout_count": 66, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 215}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 138.211, "sample_count": 2252, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 248358, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T01:39:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.066, "last_frame_pushed_at": "2026-07-02T01:39:22+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-02T01:39:22+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-02T01:39:22.950419+09:00", "mtime_epoch": 1782923962.9504192, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 117968}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T01:39:23+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 983.336, "name": "celery", "pid": 237, "rss_mb": 980.32, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 93, "pushed_frames": 93, "read_frames": 94, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.101, "pushed": 3.101, "read": 3.134}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.759, "count": 93, "max_ms": 5.693, "p95_ms": 4.069}, "gstreamer_push": {"avg_ms": 1.592, "count": 93, "max_ms": 6.301, "p95_ms": 3.819}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.016, "count": 93, "max_ms": 78.771, "p95_ms": 76.345}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.663, "count": 93, "max_ms": 87.767, "p95_ms": 86.293}, "yolo11_640_fire_11class": {"avg_ms": 61.723, "count": 93, "max_ms": 77.667, "p95_ms": 76.249}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.802, "count": 93, "max_ms": 90.676, "p95_ms": 88.219}}, "truncated_count": 0}, "processing_total": {"avg_ms": 100.989, "count": 93, "max_ms": 140.5, "p95_ms": 137.673}, "read_interval": {"avg_ms": 319.597, "count": 94, "max_ms": 19304.249, "p95_ms": 150.582}, "triton_wait": {"avg_ms": 70.633, "count": 93, "max_ms": 90.689, "p95_ms": 88.232}, "visualize": {"avg_ms": 4.36, "count": 93, "max_ms": 5.578, "p95_ms": 5.23}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2069}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4253.867, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T01:39:22+09:00", "last_error_summary": "[h264 @ 0x555d4c78b580] left block unavailable for requested intra mode [h264 @ 0x555d4c78b580] error while decoding MB 0 39, bytestream 28138 [h264 @ 0x555d4c9f7f80] error while decoding MB 20 15, bytestream -13 [in#0/rtsp @ 0x555d4bde1e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555d4bde1e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 189, "h264_decode_error_count": 308, "h264_decode_error_count_per_hour": 49, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T01:39:22+09:00", "last_error_summary": "[h264 @ 0x555d4c78b580] left block unavailable for requested intra mode [h264 @ 0x555d4c78b580] error while decoding MB 0 39, bytestream 28138 [h264 @ 0x555d4c9f7f80] error while decoding MB 20 15, bytestream -13 [in#0/rtsp @ 0x555d4bde1e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555d4bde1e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T01:39:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 159.571, "last_started_at": "2026-07-02T01:36:43+09:00", "pid": 32225, "process": {"exists": false, "pid": 32225}, "restart_count": 214, "restart_count_per_hour": 8, "rtsp_timeout_count": 66, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 215}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 138.211, "sample_count": 2252, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 248359, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T01:39:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-02T01:39:23+09:00", "last_frame_read_age_sec": 0.109, "last_frame_read_at": "2026-07-02T01:39:22+09:00", "mjpeg": {"age_sec": 0.102, "exists": true, "mtime": "2026-07-02T01:39:22.950419+09:00", "mtime_epoch": 1782923962.9504192, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 117968}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T01:39:23+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 983.336, "name": "celery", "pid": 237, "rss_mb": 980.32, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 94, "pushed_frames": 94, "read_frames": 93, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.138, "pushed": 3.138, "read": 3.105}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.747, "count": 94, "max_ms": 5.693, "p95_ms": 4.069}, "gstreamer_push": {"avg_ms": 1.59, "count": 94, "max_ms": 6.301, "p95_ms": 3.819}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.157, "count": 94, "max_ms": 78.771, "p95_ms": 76.345}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.797, "count": 94, "max_ms": 87.767, "p95_ms": 86.293}, "yolo11_640_fire_11class": {"avg_ms": 61.859, "count": 94, "max_ms": 77.667, "p95_ms": 76.249}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.983, "count": 94, "max_ms": 90.676, "p95_ms": 88.219}}, "truncated_count": 0}, "processing_total": {"avg_ms": 100.951, "count": 94, "max_ms": 140.5, "p95_ms": 137.673}, "read_interval": {"avg_ms": 321.902, "count": 93, "max_ms": 19304.249, "p95_ms": 150.582}, "triton_wait": {"avg_ms": 70.806, "count": 94, "max_ms": 90.689, "p95_ms": 88.232}, "visualize": {"avg_ms": 4.354, "count": 94, "max_ms": 5.578, "p95_ms": 5.23}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.953, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2069}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4253.805, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 189, "h264_decode_error_count": 308, "h264_decode_error_count_per_hour": 49, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T01:39:22+09:00", "last_error_summary": "[h264 @ 0x555d4c78b580] left block unavailable for requested intra mode [h264 @ 0x555d4c78b580] error while decoding MB 0 39, bytestream 28138 [h264 @ 0x555d4c9f7f80] error while decoding MB 20 15, bytestream -13 [in#0/rtsp @ 0x555d4bde1e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555d4bde1e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T01:39:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 159.578, "last_started_at": "2026-07-02T01:36:43+09:00", "pid": 32225, "process": {"exists": false, "pid": 32225}, "restart_count": 214, "restart_count_per_hour": 8, "rtsp_timeout_count": 66, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 215}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 138.211, "sample_count": 2252, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 248359, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T01:39:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.017, "last_frame_pushed_at": "2026-07-02T01:39:23+09:00", "last_frame_read_age_sec": 0.116, "last_frame_read_at": "2026-07-02T01:39:22+09:00", "mjpeg": {"age_sec": 0.109, "exists": true, "mtime": "2026-07-02T01:39:22.950419+09:00", "mtime_epoch": 1782923962.9504192, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 117968}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T01:39:23+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 983.336, "name": "celery", "pid": 237, "rss_mb": 980.32, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 94, "pushed_frames": 94, "read_frames": 93, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.137, "pushed": 3.137, "read": 3.104}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.747, "count": 94, "max_ms": 5.693, "p95_ms": 4.069}, "gstreamer_push": {"avg_ms": 1.59, "count": 94, "max_ms": 6.301, "p95_ms": 3.819}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.157, "count": 94, "max_ms": 78.771, "p95_ms": 76.345}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.797, "count": 94, "max_ms": 87.767, "p95_ms": 86.293}, "yolo11_640_fire_11class": {"avg_ms": 61.859, "count": 94, "max_ms": 77.667, "p95_ms": 76.249}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.983, "count": 94, "max_ms": 90.676, "p95_ms": 88.219}}, "truncated_count": 0}, "processing_total": {"avg_ms": 100.951, "count": 94, "max_ms": 140.5, "p95_ms": 137.673}, "read_interval": {"avg_ms": 321.902, "count": 93, "max_ms": 19304.249, "p95_ms": 150.582}, "triton_wait": {"avg_ms": 70.806, "count": 94, "max_ms": 90.689, "p95_ms": 88.232}, "visualize": {"avg_ms": 4.354, "count": 94, "max_ms": 5.578, "p95_ms": 5.23}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.96, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2069}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4358.824, "limit_mb": 8796093022207.996}, "detail": {"pid": 32332, "restart_count": 215, "start_count": 216}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 189, "h264_decode_error_count": 308, "h264_decode_error_count_per_hour": 49, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T01:39:22+09:00", "last_error_summary": "[h264 @ 0x555d4c78b580] left block unavailable for requested intra mode [h264 @ 0x555d4c78b580] error while decoding MB 0 39, bytestream 28138 [h264 @ 0x555d4c9f7f80] error while decoding MB 20 15, bytestream -13 [in#0/rtsp @ 0x555d4bde1e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555d4bde1e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T01:39:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T01:39:25+09:00", "pid": 32332, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.125, "name": "ffmpeg", "pid": 32332, "rss_mb": 139.125, "state": "S (sleeping)", "threads": 106}, "restart_count": 215, "restart_count_per_hour": 9, "rtsp_timeout_count": 66, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 216}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 138.266, "sample_count": 2206, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 248359, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T01:39:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.8, "last_frame_pushed_at": "2026-07-02T01:39:23+09:00", "last_frame_read_age_sec": 4.9, "last_frame_read_at": "2026-07-02T01:39:22+09:00", "mjpeg": {"age_sec": 4.785, "exists": true, "mtime": "2026-07-02T01:39:23.058421+09:00", "mtime_epoch": 1782923963.0584214, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 116583}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T01:39:27+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 983.336, "name": "celery", "pid": 237, "rss_mb": 980.32, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 51, "pushed_frames": 51, "read_frames": 50, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.7, "pushed": 1.7, "read": 1.667}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.086, "count": 50, "max_ms": 4.069, "p95_ms": 4.063}, "gstreamer_push": {"avg_ms": 1.757, "count": 51, "max_ms": 6.301, "p95_ms": 4.165}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.912, "count": 50, "max_ms": 77.679, "p95_ms": 75.832}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.674, "count": 50, "max_ms": 87.606, "p95_ms": 83.692}, "yolo11_640_fire_11class": {"avg_ms": 61.246, "count": 50, "max_ms": 77.667, "p95_ms": 76.218}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.757, "count": 50, "max_ms": 89.679, "p95_ms": 88.892}}, "truncated_count": 0}, "processing_total": {"avg_ms": 106.097, "count": 51, "max_ms": 140.5, "p95_ms": 138.333}, "read_interval": {"avg_ms": 504.104, "count": 50, "max_ms": 19304.249, "p95_ms": 153.157}, "triton_wait": {"avg_ms": 69.384, "count": 50, "max_ms": 89.691, "p95_ms": 88.905}, "visualize": {"avg_ms": 4.555, "count": 50, "max_ms": 5.235, "p95_ms": 5.23}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2089}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4254.945, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 8, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5577bf4b9f40] left block unavailable for requested intra mode [h264 @ 0x5577bf4b9f40] error while decoding MB 0 19, bytestream 6536 [h264 @ 0x5577bf4fcd80] error while decoding MB 62 27, bytestream -6 [h264 @ 0x5577bf5aa440] left block unavailable for requested intra mode [h264 @ 0x5577bf5aa440] error while decoding MB 0 29, bytestream 61867 [h264 @ 0x5577bf5be600] error while decoding MB 36 26, bytestream -16 [h264 @ 0x5577bf5d27c0] left block unavailable for requested intra mode [h26", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 190, "h264_decode_error_count": 316, "h264_decode_error_count_per_hour": 57, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T01:44:22+09:00", "last_error_summary": "[h264 @ 0x5577bf4b9f40] left block unavailable for requested intra mode [h264 @ 0x5577bf4b9f40] error while decoding MB 0 19, bytestream 6536 [h264 @ 0x5577bf4fcd80] error while decoding MB 62 27, bytestream -6 [h264 @ 0x5577bf5aa440] left block unavailable for requested intra mode [h264 @ 0x5577bf5aa440] error while decoding MB 0 29, bytestream 61867 [h264 @ 0x5577bf5be600] error while decoding MB 36 26, bytestream -16 [h264 @ 0x5577bf5d27c0] left block unavailable for requested intra mode [h26", "last_exit_at": "2026-07-02T01:39:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 296.886, "last_started_at": "2026-07-02T01:39:25+09:00", "pid": 32332, "process": {"exists": false, "pid": 32332}, "restart_count": 215, "restart_count_per_hour": 9, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 216}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.896, "sample_count": 2399, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 250755, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T01:44:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-02T01:44:22+09:00", "last_frame_read_age_sec": 0.059, "last_frame_read_at": "2026-07-02T01:44:22+09:00", "mjpeg": {"age_sec": 0.061, "exists": true, "mtime": "2026-07-02T01:44:22.667844+09:00", "mtime_epoch": 1782924262.6678443, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 112760}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T01:44:22+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 984.863, "name": "celery", "pid": 237, "rss_mb": 979.352, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 99, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.305, "pushed": 3.305, "read": 3.305}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.731, "count": 99, "max_ms": 4.445, "p95_ms": 3.881}, "gstreamer_push": {"avg_ms": 1.586, "count": 99, "max_ms": 5.93, "p95_ms": 3.801}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.103, "count": 99, "max_ms": 78.814, "p95_ms": 76.767}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.795, "count": 99, "max_ms": 82.202, "p95_ms": 79.658}, "yolo11_640_fire_11class": {"avg_ms": 68.484, "count": 99, "max_ms": 83.749, "p95_ms": 78.306}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.811, "count": 99, "max_ms": 93.039, "p95_ms": 89.988}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.243, "count": 99, "max_ms": 144.887, "p95_ms": 106.428}, "read_interval": {"avg_ms": 302.638, "count": 99, "max_ms": 19428.476, "p95_ms": 122.231}, "triton_wait": {"avg_ms": 78.056, "count": 99, "max_ms": 93.052, "p95_ms": 90.001}, "visualize": {"avg_ms": 3.808, "count": 99, "max_ms": 5.397, "p95_ms": 5.084}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.957, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2089}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4254.391, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T01:44:22+09:00", "last_error_summary": "[h264 @ 0x5577bf4b9f40] left block unavailable for requested intra mode [h264 @ 0x5577bf4b9f40] error while decoding MB 0 19, bytestream 6536 [h264 @ 0x5577bf4fcd80] error while decoding MB 62 27, bytestream -6 [h264 @ 0x5577bf5aa440] left block unavailable for requested intra mode [h264 @ 0x5577bf5aa440] error while decoding MB 0 29, bytestream 61867 [h264 @ 0x5577bf5be600] error while decoding MB 36 26, bytestream -16 [h264 @ 0x5577bf5d27c0] left block unavailable for requested intra mode [h26", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 190, "h264_decode_error_count": 316, "h264_decode_error_count_per_hour": 57, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T01:44:22+09:00", "last_error_summary": "[h264 @ 0x5577bf4b9f40] left block unavailable for requested intra mode [h264 @ 0x5577bf4b9f40] error while decoding MB 0 19, bytestream 6536 [h264 @ 0x5577bf4fcd80] error while decoding MB 62 27, bytestream -6 [h264 @ 0x5577bf5aa440] left block unavailable for requested intra mode [h264 @ 0x5577bf5aa440] error while decoding MB 0 29, bytestream 61867 [h264 @ 0x5577bf5be600] error while decoding MB 36 26, bytestream -16 [h264 @ 0x5577bf5d27c0] left block unavailable for requested intra mode [h26", "last_exit_at": "2026-07-02T01:44:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 296.935, "last_started_at": "2026-07-02T01:39:25+09:00", "pid": 32332, "process": {"exists": false, "pid": 32332}, "restart_count": 215, "restart_count_per_hour": 9, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 216}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.896, "sample_count": 2400, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 250756, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T01:44:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-02T01:44:22+09:00", "last_frame_read_age_sec": 0.107, "last_frame_read_at": "2026-07-02T01:44:22+09:00", "mjpeg": {"age_sec": 0.109, "exists": true, "mtime": "2026-07-02T01:44:22.667844+09:00", "mtime_epoch": 1782924262.6678443, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 112760}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T01:44:22+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 984.863, "name": "celery", "pid": 237, "rss_mb": 979.352, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 99, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.301, "pushed": 3.301, "read": 3.301}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.73, "count": 99, "max_ms": 4.445, "p95_ms": 3.881}, "gstreamer_push": {"avg_ms": 1.587, "count": 99, "max_ms": 5.93, "p95_ms": 3.801}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.36, "count": 99, "max_ms": 78.814, "p95_ms": 76.767}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.078, "count": 99, "max_ms": 82.202, "p95_ms": 79.658}, "yolo11_640_fire_11class": {"avg_ms": 68.742, "count": 99, "max_ms": 83.749, "p95_ms": 78.306}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.158, "count": 99, "max_ms": 93.039, "p95_ms": 89.988}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.591, "count": 99, "max_ms": 144.887, "p95_ms": 106.428}, "read_interval": {"avg_ms": 302.638, "count": 99, "max_ms": 19428.476, "p95_ms": 122.231}, "triton_wait": {"avg_ms": 78.403, "count": 99, "max_ms": 93.052, "p95_ms": 90.001}, "visualize": {"avg_ms": 3.809, "count": 99, "max_ms": 5.397, "p95_ms": 5.084}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.99, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2089}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4254.375, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 190, "h264_decode_error_count": 316, "h264_decode_error_count_per_hour": 57, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T01:44:22+09:00", "last_error_summary": "[h264 @ 0x5577bf4b9f40] left block unavailable for requested intra mode [h264 @ 0x5577bf4b9f40] error while decoding MB 0 19, bytestream 6536 [h264 @ 0x5577bf4fcd80] error while decoding MB 62 27, bytestream -6 [h264 @ 0x5577bf5aa440] left block unavailable for requested intra mode [h264 @ 0x5577bf5aa440] error while decoding MB 0 29, bytestream 61867 [h264 @ 0x5577bf5be600] error while decoding MB 36 26, bytestream -16 [h264 @ 0x5577bf5d27c0] left block unavailable for requested intra mode [h26", "last_exit_at": "2026-07-02T01:44:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 296.943, "last_started_at": "2026-07-02T01:39:25+09:00", "pid": 32332, "process": {"exists": false, "pid": 32332}, "restart_count": 215, "restart_count_per_hour": 9, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 216}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.896, "sample_count": 2400, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 250756, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T01:44:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-02T01:44:22+09:00", "last_frame_read_age_sec": 0.116, "last_frame_read_at": "2026-07-02T01:44:22+09:00", "mjpeg": {"age_sec": 0.117, "exists": true, "mtime": "2026-07-02T01:44:22.667844+09:00", "mtime_epoch": 1782924262.6678443, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 112760}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T01:44:22+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 984.863, "name": "celery", "pid": 237, "rss_mb": 979.352, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 99, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.3, "pushed": 3.3, "read": 3.3}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.73, "count": 99, "max_ms": 4.445, "p95_ms": 3.881}, "gstreamer_push": {"avg_ms": 1.587, "count": 99, "max_ms": 5.93, "p95_ms": 3.801}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.36, "count": 99, "max_ms": 78.814, "p95_ms": 76.767}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.078, "count": 99, "max_ms": 82.202, "p95_ms": 79.658}, "yolo11_640_fire_11class": {"avg_ms": 68.742, "count": 99, "max_ms": 83.749, "p95_ms": 78.306}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.158, "count": 99, "max_ms": 93.039, "p95_ms": 89.988}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.591, "count": 99, "max_ms": 144.887, "p95_ms": 106.428}, "read_interval": {"avg_ms": 302.638, "count": 99, "max_ms": 19428.476, "p95_ms": 122.231}, "triton_wait": {"avg_ms": 78.403, "count": 99, "max_ms": 93.052, "p95_ms": 90.001}, "visualize": {"avg_ms": 3.809, "count": 99, "max_ms": 5.397, "p95_ms": 5.084}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2089}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4359.5, "limit_mb": 8796093022207.996}, "detail": {"pid": 32439, "restart_count": 216, "start_count": 217}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 190, "h264_decode_error_count": 316, "h264_decode_error_count_per_hour": 57, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T01:44:22+09:00", "last_error_summary": "[h264 @ 0x5577bf4b9f40] left block unavailable for requested intra mode [h264 @ 0x5577bf4b9f40] error while decoding MB 0 19, bytestream 6536 [h264 @ 0x5577bf4fcd80] error while decoding MB 62 27, bytestream -6 [h264 @ 0x5577bf5aa440] left block unavailable for requested intra mode [h264 @ 0x5577bf5aa440] error while decoding MB 0 29, bytestream 61867 [h264 @ 0x5577bf5be600] error while decoding MB 36 26, bytestream -16 [h264 @ 0x5577bf5d27c0] left block unavailable for requested intra mode [h26", "last_exit_at": "2026-07-02T01:44:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T01:44:25+09:00", "pid": 32439, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.281, "name": "ffmpeg", "pid": 32439, "rss_mb": 139.281, "state": "S (sleeping)", "threads": 106}, "restart_count": 216, "restart_count_per_hour": 10, "rtsp_timeout_count": 68, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": true, "start_count": 217}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.896, "sample_count": 2397, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 250756, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T01:44:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.435, "last_frame_pushed_at": "2026-07-02T01:44:22+09:00", "last_frame_read_age_sec": 4.534, "last_frame_read_at": "2026-07-02T01:44:22+09:00", "mjpeg": {"age_sec": 4.416, "exists": true, "mtime": "2026-07-02T01:44:22.787846+09:00", "mtime_epoch": 1782924262.7878456, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 113222}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T01:44:27+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 984.863, "name": "celery", "pid": 237, "rss_mb": 979.352, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 58, "pushed_frames": 58, "read_frames": 57, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.936, "pushed": 1.936, "read": 1.903}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.819, "count": 58, "max_ms": 4.445, "p95_ms": 3.929}, "gstreamer_push": {"avg_ms": 1.591, "count": 58, "max_ms": 5.93, "p95_ms": 3.969}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.541, "count": 58, "max_ms": 78.814, "p95_ms": 77.013}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.51, "count": 58, "max_ms": 82.202, "p95_ms": 80.129}, "yolo11_640_fire_11class": {"avg_ms": 68.366, "count": 58, "max_ms": 83.749, "p95_ms": 80.349}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.164, "count": 58, "max_ms": 93.039, "p95_ms": 90.551}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.484, "count": 58, "max_ms": 144.887, "p95_ms": 111.38}, "read_interval": {"avg_ms": 447.722, "count": 57, "max_ms": 19428.476, "p95_ms": 153.351}, "triton_wait": {"avg_ms": 77.403, "count": 58, "max_ms": 93.052, "p95_ms": 90.564}, "visualize": {"avg_ms": 3.88, "count": 58, "max_ms": 5.397, "p95_ms": 5.177}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.957, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2109}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4257.551, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 6, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55ba1b7aa680] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ba1b7aa680] error while decoding MB 0 16, bytestream 7496 [h264 @ 0x55ba1c019380] error while decoding MB 13 8, bytestream -5 [h264 @ 0x55ba1b695c80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ba1b695c80] error while decoding MB 0 22, bytestream 10038 [h264 @ 0x55ba1b6d5080] error while decoding MB 77 12, bytestream -7 [h264 @ 0x55ba1c354cc0] left block unavailable for requested intra", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 191, "h264_decode_error_count": 322, "h264_decode_error_count_per_hour": 55, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T01:49:14+09:00", "last_error_summary": "[h264 @ 0x55ba1b7aa680] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ba1b7aa680] error while decoding MB 0 16, bytestream 7496 [h264 @ 0x55ba1c019380] error while decoding MB 13 8, bytestream -5 [h264 @ 0x55ba1b695c80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ba1b695c80] error while decoding MB 0 22, bytestream 10038 [h264 @ 0x55ba1b6d5080] error while decoding MB 77 12, bytestream -7 [h264 @ 0x55ba1c354cc0] left block unavailable for requested intra", "last_exit_at": "2026-07-02T01:44:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 289.604, "last_started_at": "2026-07-02T01:44:25+09:00", "pid": 32439, "process": {"exists": false, "pid": 32439}, "restart_count": 216, "restart_count_per_hour": 9, "rtsp_timeout_count": 70, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 217}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.961, "sample_count": 2390, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 253136, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T01:49:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.072, "last_frame_pushed_at": "2026-07-02T01:49:14+09:00", "last_frame_read_age_sec": 0.06, "last_frame_read_at": "2026-07-02T01:49:14+09:00", "mjpeg": {"age_sec": 0.056, "exists": true, "mtime": "2026-07-02T01:49:14.751329+09:00", "mtime_epoch": 1782924554.7513292, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 149328}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T01:49:14+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 988.023, "name": "celery", "pid": 237, "rss_mb": 979.031, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 95, "pushed_frames": 95, "read_frames": 95, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.168, "pushed": 3.168, "read": 3.168}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.491, "count": 96, "max_ms": 4.515, "p95_ms": 4.017}, "gstreamer_push": {"avg_ms": 1.402, "count": 95, "max_ms": 6.422, "p95_ms": 1.457}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.415, "count": 96, "max_ms": 78.023, "p95_ms": 70.861}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.494, "count": 96, "max_ms": 86.228, "p95_ms": 84.015}, "yolo11_640_fire_11class": {"avg_ms": 63.359, "count": 96, "max_ms": 79.822, "p95_ms": 75.784}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.274, "count": 96, "max_ms": 87.613, "p95_ms": 85.607}}, "truncated_count": 0}, "processing_total": {"avg_ms": 100.149, "count": 95, "max_ms": 138.165, "p95_ms": 136.207}, "read_interval": {"avg_ms": 316.036, "count": 95, "max_ms": 19209.606, "p95_ms": 150.81}, "triton_wait": {"avg_ms": 74.283, "count": 96, "max_ms": 87.626, "p95_ms": 85.619}, "visualize": {"avg_ms": 4.202, "count": 96, "max_ms": 6.141, "p95_ms": 5.18}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.986, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2109}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4257.488, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T01:49:14+09:00", "last_error_summary": "[h264 @ 0x55ba1b7aa680] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ba1b7aa680] error while decoding MB 0 16, bytestream 7496 [h264 @ 0x55ba1c019380] error while decoding MB 13 8, bytestream -5 [h264 @ 0x55ba1b695c80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ba1b695c80] error while decoding MB 0 22, bytestream 10038 [h264 @ 0x55ba1b6d5080] error while decoding MB 77 12, bytestream -7 [h264 @ 0x55ba1c354cc0] left block unavailable for requested intra", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 191, "h264_decode_error_count": 322, "h264_decode_error_count_per_hour": 55, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T01:49:14+09:00", "last_error_summary": "[h264 @ 0x55ba1b7aa680] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ba1b7aa680] error while decoding MB 0 16, bytestream 7496 [h264 @ 0x55ba1c019380] error while decoding MB 13 8, bytestream -5 [h264 @ 0x55ba1b695c80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ba1b695c80] error while decoding MB 0 22, bytestream 10038 [h264 @ 0x55ba1b6d5080] error while decoding MB 77 12, bytestream -7 [h264 @ 0x55ba1c354cc0] left block unavailable for requested intra", "last_exit_at": "2026-07-02T01:49:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 289.622, "last_started_at": "2026-07-02T01:44:25+09:00", "pid": 32439, "process": {"exists": false, "pid": 32439}, "restart_count": 216, "restart_count_per_hour": 9, "rtsp_timeout_count": 70, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 217}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.935, "sample_count": 2391, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 253136, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T01:49:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T01:49:14+09:00", "last_frame_read_age_sec": 0.077, "last_frame_read_at": "2026-07-02T01:49:14+09:00", "mjpeg": {"age_sec": 0.073, "exists": true, "mtime": "2026-07-02T01:49:14.751329+09:00", "mtime_epoch": 1782924554.7513292, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 149328}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T01:49:14+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 988.023, "name": "celery", "pid": 237, "rss_mb": 979.031, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 95, "pushed_frames": 95, "read_frames": 95, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.168, "pushed": 3.168, "read": 3.168}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.5, "count": 95, "max_ms": 4.515, "p95_ms": 4.017}, "gstreamer_push": {"avg_ms": 1.403, "count": 95, "max_ms": 6.422, "p95_ms": 1.457}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.212, "count": 95, "max_ms": 78.023, "p95_ms": 69.78}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.34, "count": 95, "max_ms": 86.228, "p95_ms": 83.634}, "yolo11_640_fire_11class": {"avg_ms": 63.266, "count": 95, "max_ms": 79.822, "p95_ms": 75.784}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.132, "count": 95, "max_ms": 87.613, "p95_ms": 85.285}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.85, "count": 95, "max_ms": 138.165, "p95_ms": 136.207}, "read_interval": {"avg_ms": 316.036, "count": 95, "max_ms": 19209.606, "p95_ms": 150.81}, "triton_wait": {"avg_ms": 74.15, "count": 95, "max_ms": 87.626, "p95_ms": 85.372}, "visualize": {"avg_ms": 4.209, "count": 95, "max_ms": 6.141, "p95_ms": 5.18}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.988, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2109}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4257.531, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 191, "h264_decode_error_count": 322, "h264_decode_error_count_per_hour": 55, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T01:49:14+09:00", "last_error_summary": "[h264 @ 0x55ba1b7aa680] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ba1b7aa680] error while decoding MB 0 16, bytestream 7496 [h264 @ 0x55ba1c019380] error while decoding MB 13 8, bytestream -5 [h264 @ 0x55ba1b695c80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ba1b695c80] error while decoding MB 0 22, bytestream 10038 [h264 @ 0x55ba1b6d5080] error while decoding MB 77 12, bytestream -7 [h264 @ 0x55ba1c354cc0] left block unavailable for requested intra", "last_exit_at": "2026-07-02T01:49:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 289.63, "last_started_at": "2026-07-02T01:44:25+09:00", "pid": 32439, "process": {"exists": false, "pid": 32439}, "restart_count": 216, "restart_count_per_hour": 9, "rtsp_timeout_count": 70, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 217}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.935, "sample_count": 2391, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 253136, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T01:49:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-02T01:49:14+09:00", "last_frame_read_age_sec": 0.086, "last_frame_read_at": "2026-07-02T01:49:14+09:00", "mjpeg": {"age_sec": 0.082, "exists": true, "mtime": "2026-07-02T01:49:14.751329+09:00", "mtime_epoch": 1782924554.7513292, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 149328}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T01:49:14+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 988.023, "name": "celery", "pid": 237, "rss_mb": 979.031, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 95, "pushed_frames": 95, "read_frames": 95, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.167, "pushed": 3.167, "read": 3.167}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.5, "count": 95, "max_ms": 4.515, "p95_ms": 4.017}, "gstreamer_push": {"avg_ms": 1.403, "count": 95, "max_ms": 6.422, "p95_ms": 1.457}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.212, "count": 95, "max_ms": 78.023, "p95_ms": 69.78}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.34, "count": 95, "max_ms": 86.228, "p95_ms": 83.634}, "yolo11_640_fire_11class": {"avg_ms": 63.266, "count": 95, "max_ms": 79.822, "p95_ms": 75.784}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.132, "count": 95, "max_ms": 87.613, "p95_ms": 85.285}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.85, "count": 95, "max_ms": 138.165, "p95_ms": 136.207}, "read_interval": {"avg_ms": 316.036, "count": 95, "max_ms": 19209.606, "p95_ms": 150.81}, "triton_wait": {"avg_ms": 74.15, "count": 95, "max_ms": 87.626, "p95_ms": 85.372}, "visualize": {"avg_ms": 4.209, "count": 95, "max_ms": 6.141, "p95_ms": 5.18}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2109}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4367.172, "limit_mb": 8796093022207.996}, "detail": {"pid": 32546, "restart_count": 217, "start_count": 218}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 191, "h264_decode_error_count": 322, "h264_decode_error_count_per_hour": 55, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T01:49:14+09:00", "last_error_summary": "[h264 @ 0x55ba1b7aa680] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ba1b7aa680] error while decoding MB 0 16, bytestream 7496 [h264 @ 0x55ba1c019380] error while decoding MB 13 8, bytestream -5 [h264 @ 0x55ba1b695c80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ba1b695c80] error while decoding MB 0 22, bytestream 10038 [h264 @ 0x55ba1b6d5080] error while decoding MB 77 12, bytestream -7 [h264 @ 0x55ba1c354cc0] left block unavailable for requested intra", "last_exit_at": "2026-07-02T01:49:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T01:49:17+09:00", "pid": 32546, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.645, "name": "ffmpeg", "pid": 32546, "rss_mb": 141.645, "state": "S (sleeping)", "threads": 106}, "restart_count": 217, "restart_count_per_hour": 10, "rtsp_timeout_count": 70, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": true, "start_count": 218}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.935, "sample_count": 2391, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 253136, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T01:49:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.847, "last_frame_pushed_at": "2026-07-02T01:49:14+09:00", "last_frame_read_age_sec": 4.918, "last_frame_read_at": "2026-07-02T01:49:14+09:00", "mjpeg": {"age_sec": 4.83, "exists": true, "mtime": "2026-07-02T01:49:14.835330+09:00", "mtime_epoch": 1782924554.8353302, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 147730}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T01:49:19+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 988.023, "name": "celery", "pid": 237, "rss_mb": 979.031, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 58, "pushed_frames": 58, "read_frames": 57, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.94, "pushed": 1.94, "read": 1.907}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.731, "count": 58, "max_ms": 4.408, "p95_ms": 3.881}, "gstreamer_push": {"avg_ms": 1.552, "count": 58, "max_ms": 6.422, "p95_ms": 3.825}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.44, "count": 58, "max_ms": 78.023, "p95_ms": 66.247}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.218, "count": 58, "max_ms": 83.125, "p95_ms": 81.764}, "yolo11_640_fire_11class": {"avg_ms": 68.155, "count": 58, "max_ms": 79.822, "p95_ms": 77.432}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.694, "count": 58, "max_ms": 87.613, "p95_ms": 85.285}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.179, "count": 58, "max_ms": 125.998, "p95_ms": 114.406}, "read_interval": {"avg_ms": 440.356, "count": 57, "max_ms": 19209.606, "p95_ms": 133.088}, "triton_wait": {"avg_ms": 76.359, "count": 58, "max_ms": 87.626, "p95_ms": 85.297}, "visualize": {"avg_ms": 3.739, "count": 58, "max_ms": 5.106, "p95_ms": 5.002}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.892, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2140}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4260.535, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 6, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x56075f7e8940] cabac decode of qscale diff failed at 10 24 [h264 @ 0x56075f7e8940] error while decoding MB 10 24, bytestream 41242 [h264 @ 0x56076014f880] error while decoding MB 68 39, bytestream -9 [h264 @ 0x56075f7e8940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56075f7e8940] error while decoding MB 0 4, bytestream 5132 [h264 @ 0x56076014f880] error while decoding MB 78 34, bytestream -6 [h264 @ 0x56076014f880] error while decoding MB 42 36, bytestream -7 [in#0/", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 192, "h264_decode_error_count": 328, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T01:56:39+09:00", "last_error_summary": "[h264 @ 0x56075f7e8940] cabac decode of qscale diff failed at 10 24 [h264 @ 0x56075f7e8940] error while decoding MB 10 24, bytestream 41242 [h264 @ 0x56076014f880] error while decoding MB 68 39, bytestream -9 [h264 @ 0x56075f7e8940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56075f7e8940] error while decoding MB 0 4, bytestream 5132 [h264 @ 0x56076014f880] error while decoding MB 78 34, bytestream -6 [h264 @ 0x56076014f880] error while decoding MB 42 36, bytestream -7 [in#0/", "last_exit_at": "2026-07-02T01:49:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 441.612, "last_started_at": "2026-07-02T01:49:17+09:00", "pid": 32546, "process": {"exists": false, "pid": 32546}, "restart_count": 217, "restart_count_per_hour": 8, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 218}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.751, "sample_count": 2475, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 256856, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T01:56:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.067, "last_frame_pushed_at": "2026-07-02T01:56:39+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-02T01:56:39+09:00", "mjpeg": {"age_sec": 0.051, "exists": true, "mtime": "2026-07-02T01:56:39.225780+09:00", "mtime_epoch": 1782924999.22578, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 135620}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T01:56:39+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 989.086, "name": "celery", "pid": 237, "rss_mb": 982.004, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 99, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.307, "pushed": 3.307, "read": 3.307}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.563, "count": 99, "max_ms": 6.003, "p95_ms": 4.092}, "gstreamer_push": {"avg_ms": 1.475, "count": 99, "max_ms": 5.714, "p95_ms": 3.762}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.188, "count": 99, "max_ms": 75.652, "p95_ms": 72.244}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.436, "count": 99, "max_ms": 88.205, "p95_ms": 86.024}, "yolo11_640_fire_11class": {"avg_ms": 62.014, "count": 99, "max_ms": 76.218, "p95_ms": 74.255}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.383, "count": 99, "max_ms": 87.907, "p95_ms": 86.868}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.893, "count": 99, "max_ms": 148.587, "p95_ms": 138.825}, "read_interval": {"avg_ms": 302.495, "count": 99, "max_ms": 10128.088, "p95_ms": 154.014}, "triton_wait": {"avg_ms": 71.755, "count": 99, "max_ms": 88.217, "p95_ms": 86.958}, "visualize": {"avg_ms": 4.349, "count": 99, "max_ms": 9.534, "p95_ms": 5.575}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.934, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2140}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4259.551, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T01:56:39+09:00", "last_error_summary": "[h264 @ 0x56075f7e8940] cabac decode of qscale diff failed at 10 24 [h264 @ 0x56075f7e8940] error while decoding MB 10 24, bytestream 41242 [h264 @ 0x56076014f880] error while decoding MB 68 39, bytestream -9 [h264 @ 0x56075f7e8940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56075f7e8940] error while decoding MB 0 4, bytestream 5132 [h264 @ 0x56076014f880] error while decoding MB 78 34, bytestream -6 [h264 @ 0x56076014f880] error while decoding MB 42 36, bytestream -7 [in#0/", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 192, "h264_decode_error_count": 328, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T01:56:39+09:00", "last_error_summary": "[h264 @ 0x56075f7e8940] cabac decode of qscale diff failed at 10 24 [h264 @ 0x56075f7e8940] error while decoding MB 10 24, bytestream 41242 [h264 @ 0x56076014f880] error while decoding MB 68 39, bytestream -9 [h264 @ 0x56075f7e8940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56075f7e8940] error while decoding MB 0 4, bytestream 5132 [h264 @ 0x56076014f880] error while decoding MB 78 34, bytestream -6 [h264 @ 0x56076014f880] error while decoding MB 42 36, bytestream -7 [in#0/", "last_exit_at": "2026-07-02T01:56:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 441.659, "last_started_at": "2026-07-02T01:49:17+09:00", "pid": 32546, "process": {"exists": false, "pid": 32546}, "restart_count": 217, "restart_count_per_hour": 8, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 218}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.751, "sample_count": 2476, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 256857, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T01:56:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T01:56:39+09:00", "last_frame_read_age_sec": 0.102, "last_frame_read_at": "2026-07-02T01:56:39+09:00", "mjpeg": {"age_sec": 0.098, "exists": true, "mtime": "2026-07-02T01:56:39.225780+09:00", "mtime_epoch": 1782924999.22578, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 135620}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T01:56:39+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 989.086, "name": "celery", "pid": 237, "rss_mb": 982.004, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.336, "pushed": 3.336, "read": 3.302}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.553, "count": 100, "max_ms": 6.003, "p95_ms": 4.087}, "gstreamer_push": {"avg_ms": 1.474, "count": 100, "max_ms": 5.714, "p95_ms": 3.619}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.196, "count": 100, "max_ms": 75.652, "p95_ms": 72.156}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.561, "count": 100, "max_ms": 88.205, "p95_ms": 85.586}, "yolo11_640_fire_11class": {"avg_ms": 62.119, "count": 100, "max_ms": 76.218, "p95_ms": 74.151}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.502, "count": 100, "max_ms": 87.907, "p95_ms": 86.836}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.837, "count": 100, "max_ms": 148.587, "p95_ms": 137.894}, "read_interval": {"avg_ms": 302.495, "count": 99, "max_ms": 10128.088, "p95_ms": 154.014}, "triton_wait": {"avg_ms": 71.867, "count": 100, "max_ms": 88.217, "p95_ms": 86.877}, "visualize": {"avg_ms": 4.342, "count": 100, "max_ms": 9.534, "p95_ms": 5.347}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.98, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2140}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4259.715, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 192, "h264_decode_error_count": 328, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T01:56:39+09:00", "last_error_summary": "[h264 @ 0x56075f7e8940] cabac decode of qscale diff failed at 10 24 [h264 @ 0x56075f7e8940] error while decoding MB 10 24, bytestream 41242 [h264 @ 0x56076014f880] error while decoding MB 68 39, bytestream -9 [h264 @ 0x56075f7e8940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56075f7e8940] error while decoding MB 0 4, bytestream 5132 [h264 @ 0x56076014f880] error while decoding MB 78 34, bytestream -6 [h264 @ 0x56076014f880] error while decoding MB 42 36, bytestream -7 [in#0/", "last_exit_at": "2026-07-02T01:56:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 441.667, "last_started_at": "2026-07-02T01:49:17+09:00", "pid": 32546, "process": {"exists": false, "pid": 32546}, "restart_count": 217, "restart_count_per_hour": 8, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 218}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.751, "sample_count": 2476, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 256857, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T01:56:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-02T01:56:39+09:00", "last_frame_read_age_sec": 0.109, "last_frame_read_at": "2026-07-02T01:56:39+09:00", "mjpeg": {"age_sec": 0.106, "exists": true, "mtime": "2026-07-02T01:56:39.225780+09:00", "mtime_epoch": 1782924999.22578, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 135620}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T01:56:39+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 989.086, "name": "celery", "pid": 237, "rss_mb": 982.004, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.335, "pushed": 3.335, "read": 3.301}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.553, "count": 100, "max_ms": 6.003, "p95_ms": 4.087}, "gstreamer_push": {"avg_ms": 1.474, "count": 100, "max_ms": 5.714, "p95_ms": 3.619}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.196, "count": 100, "max_ms": 75.652, "p95_ms": 72.156}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.561, "count": 100, "max_ms": 88.205, "p95_ms": 85.586}, "yolo11_640_fire_11class": {"avg_ms": 62.119, "count": 100, "max_ms": 76.218, "p95_ms": 74.151}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.502, "count": 100, "max_ms": 87.907, "p95_ms": 86.836}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.837, "count": 100, "max_ms": 148.587, "p95_ms": 137.894}, "read_interval": {"avg_ms": 302.495, "count": 99, "max_ms": 10128.088, "p95_ms": 154.014}, "triton_wait": {"avg_ms": 71.867, "count": 100, "max_ms": 88.217, "p95_ms": 86.877}, "visualize": {"avg_ms": 4.342, "count": 100, "max_ms": 9.534, "p95_ms": 5.347}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.988, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2140}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4364.504, "limit_mb": 8796093022207.996}, "detail": {"pid": 32653, "restart_count": 218, "start_count": 219}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 192, "h264_decode_error_count": 328, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T01:56:39+09:00", "last_error_summary": "[h264 @ 0x56075f7e8940] cabac decode of qscale diff failed at 10 24 [h264 @ 0x56075f7e8940] error while decoding MB 10 24, bytestream 41242 [h264 @ 0x56076014f880] error while decoding MB 68 39, bytestream -9 [h264 @ 0x56075f7e8940] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56075f7e8940] error while decoding MB 0 4, bytestream 5132 [h264 @ 0x56076014f880] error while decoding MB 78 34, bytestream -6 [h264 @ 0x56076014f880] error while decoding MB 42 36, bytestream -7 [in#0/", "last_exit_at": "2026-07-02T01:56:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T01:56:41+09:00", "pid": 32653, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.285, "name": "ffmpeg", "pid": 32653, "rss_mb": 136.285, "state": "S (sleeping)", "threads": 106}, "restart_count": 218, "restart_count_per_hour": 9, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 219}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.748, "sample_count": 2440, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 256857, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T01:56:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.364, "last_frame_pushed_at": "2026-07-02T01:56:39+09:00", "last_frame_read_age_sec": 4.459, "last_frame_read_at": "2026-07-02T01:56:39+09:00", "mjpeg": {"age_sec": 4.348, "exists": true, "mtime": "2026-07-02T01:56:39.333782+09:00", "mtime_epoch": 1782924999.3337817, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 136258}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T01:56:43+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 989.086, "name": "celery", "pid": 237, "rss_mb": 982.004, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 60, "pushed_frames": 60, "read_frames": 59, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.004, "pushed": 2.004, "read": 1.971}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.467, "count": 60, "max_ms": 4.149, "p95_ms": 4.001}, "gstreamer_push": {"avg_ms": 1.497, "count": 60, "max_ms": 5.714, "p95_ms": 3.619}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.161, "count": 60, "max_ms": 73.999, "p95_ms": 72.244}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.576, "count": 60, "max_ms": 88.205, "p95_ms": 86.024}, "yolo11_640_fire_11class": {"avg_ms": 65.417, "count": 60, "max_ms": 74.285, "p95_ms": 73.998}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.425, "count": 60, "max_ms": 87.15, "p95_ms": 86.64}}, "truncated_count": 0}, "processing_total": {"avg_ms": 102.326, "count": 60, "max_ms": 141.19, "p95_ms": 135.355}, "read_interval": {"avg_ms": 433.654, "count": 59, "max_ms": 10128.088, "p95_ms": 154.014}, "triton_wait": {"avg_ms": 76.153, "count": 60, "max_ms": 88.217, "p95_ms": 86.877}, "visualize": {"avg_ms": 4.239, "count": 60, "max_ms": 6.282, "p95_ms": 5.22}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.94, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2175}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4267.285, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55f4e68db900] error while decoding MB 60 7, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 193, "h264_decode_error_count": 329, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:04:36+09:00", "last_error_summary": "[h264 @ 0x55f4e68db900] error while decoding MB 60 7, bytestream -5", "last_exit_at": "2026-07-02T01:56:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 474.558, "last_started_at": "2026-07-02T01:56:41+09:00", "pid": 32653, "process": {"exists": false, "pid": 32653}, "restart_count": 218, "restart_count_per_hour": 8, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 219}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 138.539, "sample_count": 2666, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 261073, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:04:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.072, "last_frame_pushed_at": "2026-07-02T02:04:36+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-02T02:04:36+09:00", "mjpeg": {"age_sec": 0.057, "exists": true, "mtime": "2026-07-02T02:04:36.181418+09:00", "mtime_epoch": 1782925476.1814184, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 120998}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T02:04:36+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 991.375, "name": "celery", "pid": 237, "rss_mb": 986.746, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 254, "pushed_frames": 254, "read_frames": 254, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.481, "pushed": 8.481, "read": 8.481}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 2.936, "count": 254, "max_ms": 5.723, "p95_ms": 4.106}, "gstreamer_push": {"avg_ms": 1.352, "count": 254, "max_ms": 6.242, "p95_ms": 1.487}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.918, "count": 254, "max_ms": 75.57, "p95_ms": 70.309}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.81, "count": 254, "max_ms": 94.344, "p95_ms": 87.802}, "yolo11_640_fire_11class": {"avg_ms": 60.296, "count": 254, "max_ms": 78.335, "p95_ms": 74.006}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.011, "count": 254, "max_ms": 91.794, "p95_ms": 87.105}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.42, "count": 254, "max_ms": 153.7, "p95_ms": 139.646}, "read_interval": {"avg_ms": 118.211, "count": 254, "max_ms": 166.595, "p95_ms": 153.291}, "triton_wait": {"avg_ms": 70.619, "count": 254, "max_ms": 94.356, "p95_ms": 88.373}, "visualize": {"avg_ms": 4.512, "count": 254, "max_ms": 6.402, "p95_ms": 5.51}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.948, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2175}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4266.688, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T02:04:36+09:00", "last_error_summary": "[h264 @ 0x55f4e68db900] error while decoding MB 60 7, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 193, "h264_decode_error_count": 329, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:04:36+09:00", "last_error_summary": "[h264 @ 0x55f4e68db900] error while decoding MB 60 7, bytestream -5", "last_exit_at": "2026-07-02T02:04:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 474.627, "last_started_at": "2026-07-02T01:56:41+09:00", "pid": 32653, "process": {"exists": false, "pid": 32653}, "restart_count": 218, "restart_count_per_hour": 8, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 219}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 138.539, "sample_count": 2667, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 261074, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:04:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T02:04:36+09:00", "last_frame_read_age_sec": 0.13, "last_frame_read_at": "2026-07-02T02:04:36+09:00", "mjpeg": {"age_sec": 0.126, "exists": true, "mtime": "2026-07-02T02:04:36.181418+09:00", "mtime_epoch": 1782925476.1814184, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 120998}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T02:04:36+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 991.375, "name": "celery", "pid": 237, "rss_mb": 986.746, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 254, "pushed_frames": 254, "read_frames": 253, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.498, "pushed": 8.498, "read": 8.464}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 2.937, "count": 254, "max_ms": 5.723, "p95_ms": 4.108}, "gstreamer_push": {"avg_ms": 1.352, "count": 254, "max_ms": 6.242, "p95_ms": 1.487}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.896, "count": 254, "max_ms": 75.57, "p95_ms": 70.309}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.742, "count": 254, "max_ms": 94.344, "p95_ms": 87.802}, "yolo11_640_fire_11class": {"avg_ms": 60.246, "count": 254, "max_ms": 78.335, "p95_ms": 74.006}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.934, "count": 254, "max_ms": 91.794, "p95_ms": 87.105}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.357, "count": 254, "max_ms": 153.7, "p95_ms": 139.646}, "read_interval": {"avg_ms": 118.08, "count": 253, "max_ms": 166.595, "p95_ms": 153.291}, "triton_wait": {"avg_ms": 70.544, "count": 254, "max_ms": 94.356, "p95_ms": 88.373}, "visualize": {"avg_ms": 4.517, "count": 254, "max_ms": 6.402, "p95_ms": 5.527}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.89, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2175}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4266.797, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 193, "h264_decode_error_count": 329, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:04:36+09:00", "last_error_summary": "[h264 @ 0x55f4e68db900] error while decoding MB 60 7, bytestream -5", "last_exit_at": "2026-07-02T02:04:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 474.633, "last_started_at": "2026-07-02T01:56:41+09:00", "pid": 32653, "process": {"exists": false, "pid": 32653}, "restart_count": 218, "restart_count_per_hour": 8, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 219}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 138.539, "sample_count": 2667, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 261074, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:04:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-02T02:04:36+09:00", "last_frame_read_age_sec": 0.137, "last_frame_read_at": "2026-07-02T02:04:36+09:00", "mjpeg": {"age_sec": 0.133, "exists": true, "mtime": "2026-07-02T02:04:36.181418+09:00", "mtime_epoch": 1782925476.1814184, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 120998}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T02:04:36+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 991.375, "name": "celery", "pid": 237, "rss_mb": 986.746, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 254, "pushed_frames": 254, "read_frames": 253, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.496, "pushed": 8.496, "read": 8.462}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 2.937, "count": 254, "max_ms": 5.723, "p95_ms": 4.108}, "gstreamer_push": {"avg_ms": 1.352, "count": 254, "max_ms": 6.242, "p95_ms": 1.487}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.896, "count": 254, "max_ms": 75.57, "p95_ms": 70.309}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.742, "count": 254, "max_ms": 94.344, "p95_ms": 87.802}, "yolo11_640_fire_11class": {"avg_ms": 60.246, "count": 254, "max_ms": 78.335, "p95_ms": 74.006}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.934, "count": 254, "max_ms": 91.794, "p95_ms": 87.105}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.357, "count": 254, "max_ms": 153.7, "p95_ms": 139.646}, "read_interval": {"avg_ms": 118.08, "count": 253, "max_ms": 166.595, "p95_ms": 153.291}, "triton_wait": {"avg_ms": 70.544, "count": 254, "max_ms": 94.356, "p95_ms": 88.373}, "visualize": {"avg_ms": 4.517, "count": 254, "max_ms": 6.402, "p95_ms": 5.527}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.897, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2175}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4374.578, "limit_mb": 8796093022207.996}, "detail": {"pid": 32760, "restart_count": 219, "start_count": 220}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 193, "h264_decode_error_count": 329, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:04:36+09:00", "last_error_summary": "[h264 @ 0x55f4e68db900] error while decoding MB 60 7, bytestream -5", "last_exit_at": "2026-07-02T02:04:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T02:04:38+09:00", "pid": 32760, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.055, "name": "ffmpeg", "pid": 32760, "rss_mb": 139.055, "state": "S (sleeping)", "threads": 106}, "restart_count": 219, "restart_count_per_hour": 9, "rtsp_timeout_count": 72, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 220}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 138.599, "sample_count": 2629, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 261074, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:04:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.151, "last_frame_pushed_at": "2026-07-02T02:04:36+09:00", "last_frame_read_age_sec": 4.276, "last_frame_read_at": "2026-07-02T02:04:36+09:00", "mjpeg": {"age_sec": 4.136, "exists": true, "mtime": "2026-07-02T02:04:36.317421+09:00", "mtime_epoch": 1782925476.3174207, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 120402}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T02:04:40+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 991.375, "name": "celery", "pid": 237, "rss_mb": 986.746, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 219, "pushed_frames": 219, "read_frames": 218, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.327, "pushed": 7.327, "read": 7.293}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 2.918, "count": 219, "max_ms": 5.723, "p95_ms": 4.15}, "gstreamer_push": {"avg_ms": 1.355, "count": 219, "max_ms": 6.242, "p95_ms": 1.487}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.688, "count": 219, "max_ms": 75.57, "p95_ms": 71.431}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.077, "count": 219, "max_ms": 94.344, "p95_ms": 88.051}, "yolo11_640_fire_11class": {"avg_ms": 60.316, "count": 219, "max_ms": 75.853, "p95_ms": 74.006}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.131, "count": 219, "max_ms": 91.794, "p95_ms": 87.291}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.222, "count": 219, "max_ms": 146.124, "p95_ms": 139.87}, "read_interval": {"avg_ms": 118.008, "count": 218, "max_ms": 159.321, "p95_ms": 153.291}, "triton_wait": {"avg_ms": 70.804, "count": 219, "max_ms": 94.356, "p95_ms": 88.556}, "visualize": {"avg_ms": 4.518, "count": 219, "max_ms": 6.402, "p95_ms": 5.587}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.89, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2189}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4270.742, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x559be67c5c40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559be67c5c40] error while decoding MB 0 17, bytestream 4658 [h264 @ 0x559be67c5c40] error while decoding MB 45 16, bytestream -5 [in#0/rtsp @ 0x559be6782e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559be6782e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 194, "h264_decode_error_count": 331, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:08:31+09:00", "last_error_summary": "[h264 @ 0x559be67c5c40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559be67c5c40] error while decoding MB 0 17, bytestream 4658 [h264 @ 0x559be67c5c40] error while decoding MB 45 16, bytestream -5 [in#0/rtsp @ 0x559be6782e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559be6782e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T02:04:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 233.307, "last_started_at": "2026-07-02T02:04:38+09:00", "pid": 32760, "process": {"exists": false, "pid": 32760}, "restart_count": 219, "restart_count_per_hour": 9, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 220}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 140.753, "sample_count": 2227, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 262745, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:08:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.071, "last_frame_pushed_at": "2026-07-02T02:08:31+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-02T02:08:31+09:00", "mjpeg": {"age_sec": 0.054, "exists": true, "mtime": "2026-07-02T02:08:31.705316+09:00", "mtime_epoch": 1782925711.7053158, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 121267}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T02:08:31+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 994.406, "name": "celery", "pid": 237, "rss_mb": 988.785, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 85, "pushed_frames": 85, "read_frames": 85, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.838, "pushed": 2.838, "read": 2.838}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.128, "count": 85, "max_ms": 5.963, "p95_ms": 5.63}, "gstreamer_push": {"avg_ms": 1.287, "count": 85, "max_ms": 4.38, "p95_ms": 1.732}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.475, "count": 85, "max_ms": 70.344, "p95_ms": 65.354}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.626, "count": 85, "max_ms": 91.26, "p95_ms": 87.809}, "yolo11_640_fire_11class": {"avg_ms": 55.278, "count": 85, "max_ms": 76.551, "p95_ms": 70.891}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.335, "count": 85, "max_ms": 89.627, "p95_ms": 82.549}}, "truncated_count": 0}, "processing_total": {"avg_ms": 117.709, "count": 85, "max_ms": 142.735, "p95_ms": 141.462}, "read_interval": {"avg_ms": 353.017, "count": 85, "max_ms": 10127.184, "p95_ms": 156.371}, "triton_wait": {"avg_ms": 66.704, "count": 85, "max_ms": 91.273, "p95_ms": 87.821}, "visualize": {"avg_ms": 5.304, "count": 85, "max_ms": 9.085, "p95_ms": 6.561}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.951, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2189}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4269.906, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T02:08:31+09:00", "last_error_summary": "[h264 @ 0x559be67c5c40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559be67c5c40] error while decoding MB 0 17, bytestream 4658 [h264 @ 0x559be67c5c40] error while decoding MB 45 16, bytestream -5 [in#0/rtsp @ 0x559be6782e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559be6782e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 194, "h264_decode_error_count": 331, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:08:31+09:00", "last_error_summary": "[h264 @ 0x559be67c5c40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559be67c5c40] error while decoding MB 0 17, bytestream 4658 [h264 @ 0x559be67c5c40] error while decoding MB 45 16, bytestream -5 [in#0/rtsp @ 0x559be6782e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559be6782e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T02:08:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 233.376, "last_started_at": "2026-07-02T02:04:38+09:00", "pid": 32760, "process": {"exists": false, "pid": 32760}, "restart_count": 219, "restart_count_per_hour": 9, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 220}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 140.753, "sample_count": 2227, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 262746, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:08:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T02:08:31+09:00", "last_frame_read_age_sec": 0.13, "last_frame_read_at": "2026-07-02T02:08:31+09:00", "mjpeg": {"age_sec": 0.123, "exists": true, "mtime": "2026-07-02T02:08:31.705316+09:00", "mtime_epoch": 1782925711.7053158, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 121267}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T02:08:31+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 994.406, "name": "celery", "pid": 237, "rss_mb": 988.785, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 85, "pushed_frames": 85, "read_frames": 84, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.846, "pushed": 2.846, "read": 2.812}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.174, "count": 85, "max_ms": 8.134, "p95_ms": 5.663}, "gstreamer_push": {"avg_ms": 1.287, "count": 85, "max_ms": 4.38, "p95_ms": 1.732}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.51, "count": 85, "max_ms": 70.344, "p95_ms": 65.354}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.668, "count": 85, "max_ms": 91.26, "p95_ms": 87.809}, "yolo11_640_fire_11class": {"avg_ms": 55.317, "count": 85, "max_ms": 76.551, "p95_ms": 70.891}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.394, "count": 85, "max_ms": 89.627, "p95_ms": 82.549}}, "truncated_count": 0}, "processing_total": {"avg_ms": 117.773, "count": 85, "max_ms": 142.735, "p95_ms": 141.462}, "read_interval": {"avg_ms": 355.686, "count": 84, "max_ms": 10127.184, "p95_ms": 156.371}, "triton_wait": {"avg_ms": 66.746, "count": 85, "max_ms": 91.273, "p95_ms": 87.821}, "visualize": {"avg_ms": 5.308, "count": 85, "max_ms": 9.085, "p95_ms": 6.561}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.869, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2189}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4269.902, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 194, "h264_decode_error_count": 331, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:08:31+09:00", "last_error_summary": "[h264 @ 0x559be67c5c40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559be67c5c40] error while decoding MB 0 17, bytestream 4658 [h264 @ 0x559be67c5c40] error while decoding MB 45 16, bytestream -5 [in#0/rtsp @ 0x559be6782e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559be6782e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T02:08:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 233.381, "last_started_at": "2026-07-02T02:04:38+09:00", "pid": 32760, "process": {"exists": false, "pid": 32760}, "restart_count": 219, "restart_count_per_hour": 9, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 220}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 140.753, "sample_count": 2227, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 262746, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:08:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-02T02:08:31+09:00", "last_frame_read_age_sec": 0.134, "last_frame_read_at": "2026-07-02T02:08:31+09:00", "mjpeg": {"age_sec": 0.128, "exists": true, "mtime": "2026-07-02T02:08:31.705316+09:00", "mtime_epoch": 1782925711.7053158, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 121267}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T02:08:31+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 994.406, "name": "celery", "pid": 237, "rss_mb": 988.785, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 85, "pushed_frames": 85, "read_frames": 84, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.845, "pushed": 2.845, "read": 2.812}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.174, "count": 85, "max_ms": 8.134, "p95_ms": 5.663}, "gstreamer_push": {"avg_ms": 1.287, "count": 85, "max_ms": 4.38, "p95_ms": 1.732}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.51, "count": 85, "max_ms": 70.344, "p95_ms": 65.354}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.668, "count": 85, "max_ms": 91.26, "p95_ms": 87.809}, "yolo11_640_fire_11class": {"avg_ms": 55.317, "count": 85, "max_ms": 76.551, "p95_ms": 70.891}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.394, "count": 85, "max_ms": 89.627, "p95_ms": 82.549}}, "truncated_count": 0}, "processing_total": {"avg_ms": 117.773, "count": 85, "max_ms": 142.735, "p95_ms": 141.462}, "read_interval": {"avg_ms": 355.686, "count": 84, "max_ms": 10127.184, "p95_ms": 156.371}, "triton_wait": {"avg_ms": 66.746, "count": 85, "max_ms": 91.273, "p95_ms": 87.821}, "visualize": {"avg_ms": 5.308, "count": 85, "max_ms": 9.085, "p95_ms": 6.561}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.873, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2189}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4375.594, "limit_mb": 8796093022207.996}, "detail": {"pid": 32867, "restart_count": 220, "start_count": 221}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 194, "h264_decode_error_count": 331, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:08:31+09:00", "last_error_summary": "[h264 @ 0x559be67c5c40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559be67c5c40] error while decoding MB 0 17, bytestream 4658 [h264 @ 0x559be67c5c40] error while decoding MB 45 16, bytestream -5 [in#0/rtsp @ 0x559be6782e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559be6782e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T02:08:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T02:08:34+09:00", "pid": 32867, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.051, "name": "ffmpeg", "pid": 32867, "rss_mb": 139.051, "state": "S (sleeping)", "threads": 106}, "restart_count": 220, "restart_count_per_hour": 10, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 221}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 140.756, "sample_count": 2189, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 262746, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:08:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.475, "last_frame_pushed_at": "2026-07-02T02:08:31+09:00", "last_frame_read_age_sec": 4.599, "last_frame_read_at": "2026-07-02T02:08:31+09:00", "mjpeg": {"age_sec": 4.46, "exists": true, "mtime": "2026-07-02T02:08:31.837318+09:00", "mtime_epoch": 1782925711.8373182, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 120169}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T02:08:36+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 994.406, "name": "celery", "pid": 237, "rss_mb": 988.785, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 53, "pushed_frames": 53, "read_frames": 52, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.772, "pushed": 1.772, "read": 1.739}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.225, "count": 53, "max_ms": 8.134, "p95_ms": 5.917}, "gstreamer_push": {"avg_ms": 1.28, "count": 53, "max_ms": 4.38, "p95_ms": 1.732}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.118, "count": 53, "max_ms": 67.31, "p95_ms": 65.354}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.04, "count": 53, "max_ms": 86.453, "p95_ms": 84.345}, "yolo11_640_fire_11class": {"avg_ms": 52.475, "count": 53, "max_ms": 76.551, "p95_ms": 68.73}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.633, "count": 53, "max_ms": 87.252, "p95_ms": 82.051}}, "truncated_count": 0}, "processing_total": {"avg_ms": 113.169, "count": 53, "max_ms": 142.262, "p95_ms": 140.153}, "read_interval": {"avg_ms": 489.195, "count": 52, "max_ms": 10127.184, "p95_ms": 157.161}, "triton_wait": {"avg_ms": 63.157, "count": 53, "max_ms": 87.265, "p95_ms": 84.36}, "visualize": {"avg_ms": 5.317, "count": 53, "max_ms": 9.085, "p95_ms": 6.633}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.905, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2201}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4264.445, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5628103ab540] error while decoding MB 49 34, bytestream -18", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 195, "h264_decode_error_count": 332, "h264_decode_error_count_per_hour": 49, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:11:49+09:00", "last_error_summary": "[h264 @ 0x5628103ab540] error while decoding MB 49 34, bytestream -18", "last_exit_at": "2026-07-02T02:08:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 195.112, "last_started_at": "2026-07-02T02:08:34+09:00", "pid": 32867, "process": {"exists": false, "pid": 32867}, "restart_count": 220, "restart_count_per_hour": 10, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 221}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 141.74, "sample_count": 2112, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 264188, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:11:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-02T02:11:49+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-02T02:11:49+09:00", "mjpeg": {"age_sec": 0.048, "exists": true, "mtime": "2026-07-02T02:11:49.360621+09:00", "mtime_epoch": 1782925909.3606212, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 122515}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T02:11:49+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 994.828, "name": "celery", "pid": 237, "rss_mb": 982.762, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 224, "pushed_frames": 224, "read_frames": 224, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.502, "pushed": 7.502, "read": 7.502}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.963, "count": 224, "max_ms": 5.416, "p95_ms": 4.678}, "gstreamer_push": {"avg_ms": 1.284, "count": 224, "max_ms": 4.062, "p95_ms": 2.514}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.814, "count": 224, "max_ms": 74.158, "p95_ms": 66.428}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.486, "count": 224, "max_ms": 92.707, "p95_ms": 89.501}, "yolo11_640_fire_11class": {"avg_ms": 56.305, "count": 224, "max_ms": 75.041, "p95_ms": 72.481}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.226, "count": 224, "max_ms": 89.289, "p95_ms": 86.58}}, "truncated_count": 0}, "processing_total": {"avg_ms": 120.351, "count": 224, "max_ms": 147.042, "p95_ms": 142.271}, "read_interval": {"avg_ms": 133.703, "count": 224, "max_ms": 161.199, "p95_ms": 155.191}, "triton_wait": {"avg_ms": 69.615, "count": 224, "max_ms": 92.722, "p95_ms": 89.512}, "visualize": {"avg_ms": 5.083, "count": 224, "max_ms": 6.376, "p95_ms": 5.412}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.86, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2201}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4263.934, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T02:11:49+09:00", "last_error_summary": "[h264 @ 0x5628103ab540] error while decoding MB 49 34, bytestream -18", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 195, "h264_decode_error_count": 332, "h264_decode_error_count_per_hour": 49, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:11:49+09:00", "last_error_summary": "[h264 @ 0x5628103ab540] error while decoding MB 49 34, bytestream -18", "last_exit_at": "2026-07-02T02:11:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 195.196, "last_started_at": "2026-07-02T02:08:34+09:00", "pid": 32867, "process": {"exists": false, "pid": 32867}, "restart_count": 220, "restart_count_per_hour": 10, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 221}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 141.74, "sample_count": 2112, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 264189, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:11:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T02:11:49+09:00", "last_frame_read_age_sec": 0.136, "last_frame_read_at": "2026-07-02T02:11:49+09:00", "mjpeg": {"age_sec": 0.132, "exists": true, "mtime": "2026-07-02T02:11:49.360621+09:00", "mtime_epoch": 1782925909.3606212, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 122515}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T02:11:49+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 994.828, "name": "celery", "pid": 237, "rss_mb": 983.656, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 225, "pushed_frames": 225, "read_frames": 224, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.514, "pushed": 7.514, "read": 7.481}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.978, "count": 225, "max_ms": 7.208, "p95_ms": 4.8}, "gstreamer_push": {"avg_ms": 1.283, "count": 225, "max_ms": 4.062, "p95_ms": 2.514}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.764, "count": 225, "max_ms": 74.158, "p95_ms": 66.428}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.432, "count": 225, "max_ms": 92.707, "p95_ms": 89.501}, "yolo11_640_fire_11class": {"avg_ms": 56.258, "count": 225, "max_ms": 75.041, "p95_ms": 72.481}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.158, "count": 225, "max_ms": 89.289, "p95_ms": 86.58}}, "truncated_count": 0}, "processing_total": {"avg_ms": 120.39, "count": 225, "max_ms": 147.042, "p95_ms": 142.271}, "read_interval": {"avg_ms": 133.703, "count": 224, "max_ms": 161.199, "p95_ms": 155.191}, "triton_wait": {"avg_ms": 69.561, "count": 225, "max_ms": 92.722, "p95_ms": 89.512}, "visualize": {"avg_ms": 5.084, "count": 225, "max_ms": 6.376, "p95_ms": 5.412}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.943, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2201}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4263.926, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 195, "h264_decode_error_count": 332, "h264_decode_error_count_per_hour": 49, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:11:49+09:00", "last_error_summary": "[h264 @ 0x5628103ab540] error while decoding MB 49 34, bytestream -18", "last_exit_at": "2026-07-02T02:11:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 195.201, "last_started_at": "2026-07-02T02:08:34+09:00", "pid": 32867, "process": {"exists": false, "pid": 32867}, "restart_count": 220, "restart_count_per_hour": 10, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 221}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 141.74, "sample_count": 2112, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 264189, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:11:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-02T02:11:49+09:00", "last_frame_read_age_sec": 0.141, "last_frame_read_at": "2026-07-02T02:11:49+09:00", "mjpeg": {"age_sec": 0.137, "exists": true, "mtime": "2026-07-02T02:11:49.360621+09:00", "mtime_epoch": 1782925909.3606212, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 122515}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T02:11:49+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 994.828, "name": "celery", "pid": 237, "rss_mb": 983.656, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 225, "pushed_frames": 225, "read_frames": 224, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.513, "pushed": 7.513, "read": 7.48}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.978, "count": 225, "max_ms": 7.208, "p95_ms": 4.8}, "gstreamer_push": {"avg_ms": 1.283, "count": 225, "max_ms": 4.062, "p95_ms": 2.514}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.764, "count": 225, "max_ms": 74.158, "p95_ms": 66.428}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.432, "count": 225, "max_ms": 92.707, "p95_ms": 89.501}, "yolo11_640_fire_11class": {"avg_ms": 56.258, "count": 225, "max_ms": 75.041, "p95_ms": 72.481}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.158, "count": 225, "max_ms": 89.289, "p95_ms": 86.58}}, "truncated_count": 0}, "processing_total": {"avg_ms": 120.39, "count": 225, "max_ms": 147.042, "p95_ms": 142.271}, "read_interval": {"avg_ms": 133.703, "count": 224, "max_ms": 161.199, "p95_ms": 155.191}, "triton_wait": {"avg_ms": 69.561, "count": 225, "max_ms": 92.722, "p95_ms": 89.512}, "visualize": {"avg_ms": 5.084, "count": 225, "max_ms": 6.376, "p95_ms": 5.412}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.948, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2201}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4373.352, "limit_mb": 8796093022207.996}, "detail": {"pid": 32974, "restart_count": 221, "start_count": 222}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 195, "h264_decode_error_count": 332, "h264_decode_error_count_per_hour": 49, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:11:49+09:00", "last_error_summary": "[h264 @ 0x5628103ab540] error while decoding MB 49 34, bytestream -18", "last_exit_at": "2026-07-02T02:11:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T02:11:52+09:00", "pid": 32974, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.879, "name": "ffmpeg", "pid": 32974, "rss_mb": 138.879, "state": "S (sleeping)", "threads": 106}, "restart_count": 221, "restart_count_per_hour": 11, "rtsp_timeout_count": 74, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 222}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 141.754, "sample_count": 2075, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 264189, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:11:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.902, "last_frame_pushed_at": "2026-07-02T02:11:49+09:00", "last_frame_read_age_sec": 5.033, "last_frame_read_at": "2026-07-02T02:11:49+09:00", "mjpeg": {"age_sec": 4.889, "exists": true, "mtime": "2026-07-02T02:11:49.500623+09:00", "mtime_epoch": 1782925909.5006235, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 122212}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T02:11:54+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 994.828, "name": "celery", "pid": 237, "rss_mb": 983.656, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 188, "pushed_frames": 188, "read_frames": 187, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.283, "pushed": 6.283, "read": 6.249}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.006, "count": 188, "max_ms": 7.208, "p95_ms": 5.021}, "gstreamer_push": {"avg_ms": 1.303, "count": 188, "max_ms": 4.062, "p95_ms": 2.564}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.339, "count": 188, "max_ms": 74.158, "p95_ms": 66.01}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.628, "count": 188, "max_ms": 92.707, "p95_ms": 88.471}, "yolo11_640_fire_11class": {"avg_ms": 55.231, "count": 188, "max_ms": 75.041, "p95_ms": 72.054}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.291, "count": 188, "max_ms": 89.289, "p95_ms": 86.235}}, "truncated_count": 0}, "processing_total": {"avg_ms": 120.411, "count": 188, "max_ms": 147.042, "p95_ms": 142.423}, "read_interval": {"avg_ms": 133.679, "count": 187, "max_ms": 161.199, "p95_ms": 155.395}, "triton_wait": {"avg_ms": 68.739, "count": 188, "max_ms": 92.722, "p95_ms": 88.483}, "visualize": {"avg_ms": 5.115, "count": 188, "max_ms": 6.376, "p95_ms": 5.594}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.923, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2251}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4273.578, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 6, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55fa59f72ec0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55fa59f72ec0] error while decoding MB 0 17, bytestream 6371 [h264 @ 0x55fa5ac237c0] top block unavailable for requested intra mode -1 [h264 @ 0x55fa5ac237c0] error while decoding MB 42 0, bytestream 2419 [h264 @ 0x55fa5ac237c0] left block unavailable for requested intra mode [h264 @ 0x55fa5ac237c0] error while decoding MB 0 20, bytestream 39817 [h264 @ 0x55fa59f72ec0] cabac decode of qscale diff failed at 14 ", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 196, "h264_decode_error_count": 338, "h264_decode_error_count_per_hour": 51, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:22:41+09:00", "last_error_summary": "[h264 @ 0x55fa59f72ec0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55fa59f72ec0] error while decoding MB 0 17, bytestream 6371 [h264 @ 0x55fa5ac237c0] top block unavailable for requested intra mode -1 [h264 @ 0x55fa5ac237c0] error while decoding MB 42 0, bytestream 2419 [h264 @ 0x55fa5ac237c0] left block unavailable for requested intra mode [h264 @ 0x55fa5ac237c0] error while decoding MB 0 20, bytestream 39817 [h264 @ 0x55fa59f72ec0] cabac decode of qscale diff failed at 14 ", "last_exit_at": "2026-07-02T02:11:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 649.16, "last_started_at": "2026-07-02T02:11:52+09:00", "pid": 32974, "process": {"exists": false, "pid": 32974}, "restart_count": 221, "restart_count_per_hour": 10, "rtsp_timeout_count": 76, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 222}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.364, "sample_count": 2690, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 270181, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:22:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.07, "last_frame_pushed_at": "2026-07-02T02:22:41+09:00", "last_frame_read_age_sec": 0.058, "last_frame_read_at": "2026-07-02T02:22:41+09:00", "mjpeg": {"age_sec": 0.054, "exists": true, "mtime": "2026-07-02T02:22:41.494821+09:00", "mtime_epoch": 1782926561.4948206, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 136829}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T02:22:41+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 995.559, "name": "celery", "pid": 237, "rss_mb": 988.461, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 100, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.341, "pushed": 3.341, "read": 3.341}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.534, "count": 100, "max_ms": 2.169, "p95_ms": 1.564}, "gstreamer_push": {"avg_ms": 1.711, "count": 100, "max_ms": 6.39, "p95_ms": 3.847}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.26, "count": 100, "max_ms": 77.813, "p95_ms": 76.765}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.356, "count": 100, "max_ms": 88.118, "p95_ms": 85.992}, "yolo11_640_fire_11class": {"avg_ms": 72.454, "count": 100, "max_ms": 79.051, "p95_ms": 77.305}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 81.564, "count": 100, "max_ms": 90.619, "p95_ms": 86.418}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.531, "count": 100, "max_ms": 100.689, "p95_ms": 97.601}, "read_interval": {"avg_ms": 299.7, "count": 100, "max_ms": 19148.456, "p95_ms": 117.146}, "triton_wait": {"avg_ms": 82.113, "count": 100, "max_ms": 90.633, "p95_ms": 87.313}, "visualize": {"avg_ms": 3.677, "count": 100, "max_ms": 4.447, "p95_ms": 3.868}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.93, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2251}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4272.777, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T02:22:41+09:00", "last_error_summary": "[h264 @ 0x55fa59f72ec0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55fa59f72ec0] error while decoding MB 0 17, bytestream 6371 [h264 @ 0x55fa5ac237c0] top block unavailable for requested intra mode -1 [h264 @ 0x55fa5ac237c0] error while decoding MB 42 0, bytestream 2419 [h264 @ 0x55fa5ac237c0] left block unavailable for requested intra mode [h264 @ 0x55fa5ac237c0] error while decoding MB 0 20, bytestream 39817 [h264 @ 0x55fa59f72ec0] cabac decode of qscale diff failed at 14 ", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 196, "h264_decode_error_count": 338, "h264_decode_error_count_per_hour": 51, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:22:41+09:00", "last_error_summary": "[h264 @ 0x55fa59f72ec0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55fa59f72ec0] error while decoding MB 0 17, bytestream 6371 [h264 @ 0x55fa5ac237c0] top block unavailable for requested intra mode -1 [h264 @ 0x55fa5ac237c0] error while decoding MB 42 0, bytestream 2419 [h264 @ 0x55fa5ac237c0] left block unavailable for requested intra mode [h264 @ 0x55fa5ac237c0] error while decoding MB 0 20, bytestream 39817 [h264 @ 0x55fa59f72ec0] cabac decode of qscale diff failed at 14 ", "last_exit_at": "2026-07-02T02:22:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 649.205, "last_started_at": "2026-07-02T02:11:52+09:00", "pid": 32974, "process": {"exists": false, "pid": 32974}, "restart_count": 221, "restart_count_per_hour": 10, "rtsp_timeout_count": 76, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 222}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.364, "sample_count": 2690, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 270182, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:22:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T02:22:41+09:00", "last_frame_read_age_sec": 0.104, "last_frame_read_at": "2026-07-02T02:22:41+09:00", "mjpeg": {"age_sec": 0.099, "exists": true, "mtime": "2026-07-02T02:22:41.494821+09:00", "mtime_epoch": 1782926561.4948206, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 136829}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T02:22:41+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 995.559, "name": "celery", "pid": 237, "rss_mb": 988.461, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 100, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.369, "pushed": 3.369, "read": 3.336}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.532, "count": 101, "max_ms": 2.169, "p95_ms": 1.564}, "gstreamer_push": {"avg_ms": 1.708, "count": 101, "max_ms": 6.39, "p95_ms": 3.847}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.186, "count": 101, "max_ms": 77.813, "p95_ms": 76.765}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.394, "count": 101, "max_ms": 88.118, "p95_ms": 85.992}, "yolo11_640_fire_11class": {"avg_ms": 72.479, "count": 101, "max_ms": 79.051, "p95_ms": 77.305}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 81.605, "count": 101, "max_ms": 90.619, "p95_ms": 86.418}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.556, "count": 101, "max_ms": 100.689, "p95_ms": 97.601}, "read_interval": {"avg_ms": 299.7, "count": 100, "max_ms": 19148.456, "p95_ms": 117.146}, "triton_wait": {"avg_ms": 82.148, "count": 101, "max_ms": 90.633, "p95_ms": 87.313}, "visualize": {"avg_ms": 3.672, "count": 101, "max_ms": 4.447, "p95_ms": 3.868}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.976, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2251}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4272.863, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 196, "h264_decode_error_count": 338, "h264_decode_error_count_per_hour": 51, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:22:41+09:00", "last_error_summary": "[h264 @ 0x55fa59f72ec0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55fa59f72ec0] error while decoding MB 0 17, bytestream 6371 [h264 @ 0x55fa5ac237c0] top block unavailable for requested intra mode -1 [h264 @ 0x55fa5ac237c0] error while decoding MB 42 0, bytestream 2419 [h264 @ 0x55fa5ac237c0] left block unavailable for requested intra mode [h264 @ 0x55fa5ac237c0] error while decoding MB 0 20, bytestream 39817 [h264 @ 0x55fa59f72ec0] cabac decode of qscale diff failed at 14 ", "last_exit_at": "2026-07-02T02:22:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 649.213, "last_started_at": "2026-07-02T02:11:52+09:00", "pid": 32974, "process": {"exists": false, "pid": 32974}, "restart_count": 221, "restart_count_per_hour": 10, "rtsp_timeout_count": 76, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 222}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.364, "sample_count": 2690, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 270182, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:22:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-02T02:22:41+09:00", "last_frame_read_age_sec": 0.112, "last_frame_read_at": "2026-07-02T02:22:41+09:00", "mjpeg": {"age_sec": 0.107, "exists": true, "mtime": "2026-07-02T02:22:41.494821+09:00", "mtime_epoch": 1782926561.4948206, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 136829}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T02:22:41+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 995.559, "name": "celery", "pid": 237, "rss_mb": 988.461, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 100, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.369, "pushed": 3.369, "read": 3.335}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.532, "count": 101, "max_ms": 2.169, "p95_ms": 1.564}, "gstreamer_push": {"avg_ms": 1.708, "count": 101, "max_ms": 6.39, "p95_ms": 3.847}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.186, "count": 101, "max_ms": 77.813, "p95_ms": 76.765}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.394, "count": 101, "max_ms": 88.118, "p95_ms": 85.992}, "yolo11_640_fire_11class": {"avg_ms": 72.479, "count": 101, "max_ms": 79.051, "p95_ms": 77.305}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 81.605, "count": 101, "max_ms": 90.619, "p95_ms": 86.418}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.556, "count": 101, "max_ms": 100.689, "p95_ms": 97.601}, "read_interval": {"avg_ms": 299.7, "count": 100, "max_ms": 19148.456, "p95_ms": 117.146}, "triton_wait": {"avg_ms": 82.148, "count": 101, "max_ms": 90.633, "p95_ms": 87.313}, "visualize": {"avg_ms": 3.672, "count": 101, "max_ms": 4.447, "p95_ms": 3.868}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.983, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2251}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4380.707, "limit_mb": 8796093022207.996}, "detail": {"pid": 33081, "restart_count": 222, "start_count": 223}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 196, "h264_decode_error_count": 338, "h264_decode_error_count_per_hour": 51, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:22:41+09:00", "last_error_summary": "[h264 @ 0x55fa59f72ec0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55fa59f72ec0] error while decoding MB 0 17, bytestream 6371 [h264 @ 0x55fa5ac237c0] top block unavailable for requested intra mode -1 [h264 @ 0x55fa5ac237c0] error while decoding MB 42 0, bytestream 2419 [h264 @ 0x55fa5ac237c0] left block unavailable for requested intra mode [h264 @ 0x55fa5ac237c0] error while decoding MB 0 20, bytestream 39817 [h264 @ 0x55fa59f72ec0] cabac decode of qscale diff failed at 14 ", "last_exit_at": "2026-07-02T02:22:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T02:22:44+09:00", "pid": 33081, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.168, "name": "ffmpeg", "pid": 33081, "rss_mb": 139.168, "state": "S (sleeping)", "threads": 106}, "restart_count": 222, "restart_count_per_hour": 11, "rtsp_timeout_count": 76, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 223}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.955, "sample_count": 2645, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 270182, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:22:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.561, "last_frame_pushed_at": "2026-07-02T02:22:41+09:00", "last_frame_read_age_sec": 4.658, "last_frame_read_at": "2026-07-02T02:22:41+09:00", "mjpeg": {"age_sec": 4.546, "exists": true, "mtime": "2026-07-02T02:22:41.602823+09:00", "mtime_epoch": 1782926561.6028233, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 117452}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T02:22:46+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 995.559, "name": "celery", "pid": 237, "rss_mb": 988.461, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 59, "pushed_frames": 59, "read_frames": 58, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.972, "pushed": 1.972, "read": 1.939}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.523, "count": 59, "max_ms": 2.169, "p95_ms": 1.564}, "gstreamer_push": {"avg_ms": 1.681, "count": 59, "max_ms": 6.39, "p95_ms": 3.831}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.423, "count": 59, "max_ms": 77.813, "p95_ms": 76.753}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.191, "count": 59, "max_ms": 88.118, "p95_ms": 87.302}, "yolo11_640_fire_11class": {"avg_ms": 71.325, "count": 59, "max_ms": 77.917, "p95_ms": 77.305}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.74, "count": 59, "max_ms": 87.421, "p95_ms": 86.652}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.048, "count": 59, "max_ms": 99.432, "p95_ms": 97.705}, "read_interval": {"avg_ms": 437.097, "count": 58, "max_ms": 19148.456, "p95_ms": 118.149}, "triton_wait": {"avg_ms": 81.629, "count": 59, "max_ms": 88.13, "p95_ms": 87.433}, "visualize": {"avg_ms": 3.678, "count": 59, "max_ms": 4.447, "p95_ms": 4.045}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.915, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3280}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4521.016, "limit_mb": 8796093022207.996}, "detail": {"watchdog_timeout_sec": 30.0}, "event_type": "frame_read_watchdog_timeout", "ffmpeg": {"error_count": 44, "h264_decode_error_count": 45, "h264_decode_error_count_per_hour": 0, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:56:17+09:00", "last_error_summary": "[h264 @ 0x5592b9f61e80] error while decoding MB 25 47, bytestream -5 [in#0/rtsp @ 0x5592b9f1be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5592b9f1be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T23:56:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 9019.568, "last_started_at": "2026-07-01T23:56:19+09:00", "pid": 30468, "process": {"exists": true, "fd_count": 5, "hwm_mb": 328.957, "name": "ffmpeg", "pid": 30468, "rss_mb": 328.957, "state": "S (sleeping)", "threads": 106}, "restart_count": 44, "restart_count_per_hour": 0, "rtsp_timeout_count": 80, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 45}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.488, "sample_count": 2654, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 393700, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:26:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 30.06, "last_frame_pushed_at": "2026-07-02T02:26:09+09:00", "last_frame_read_age_sec": 30.155, "last_frame_read_at": "2026-07-02T02:26:09+09:00", "mjpeg": {"age_sec": 30.043, "exists": true, "mtime": "2026-07-02T02:26:09.195912+09:00", "mtime_epoch": 1782926769.1959124, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101812}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T02:26:39+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1054.961, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3280}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4505.305, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 44, "h264_decode_error_count": 45, "h264_decode_error_count_per_hour": 0, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-01T23:56:17+09:00", "last_error_summary": "[h264 @ 0x5592b9f61e80] error while decoding MB 25 47, bytestream -5 [in#0/rtsp @ 0x5592b9f1be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5592b9f1be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T23:56:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 9019.582, "last_started_at": "2026-07-01T23:56:19+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 44, "restart_count_per_hour": 0, "rtsp_timeout_count": 80, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 45}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.488, "sample_count": 2654, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 393700, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:26:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 30.074, "last_frame_pushed_at": "2026-07-02T02:26:09+09:00", "last_frame_read_age_sec": 30.17, "last_frame_read_at": "2026-07-02T02:26:09+09:00", "mjpeg": {"age_sec": 30.057, "exists": true, "mtime": "2026-07-02T02:26:09.195912+09:00", "mtime_epoch": 1782926769.1959124, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101812}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T02:26:39+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1054.961, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3280}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4220.258, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55e252358cc0] error while decoding MB 25 24, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 45, "h264_decode_error_count": 46, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:26:39+09:00", "last_error_summary": "[h264 @ 0x55e252358cc0] error while decoding MB 25 24, bytestream -5", "last_exit_at": "2026-07-01T23:56:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 9019.633, "last_started_at": "2026-07-01T23:56:19+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 44, "restart_count_per_hour": 0, "rtsp_timeout_count": 80, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 45}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.488, "sample_count": 2653, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 393700, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:26:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 30.126, "last_frame_pushed_at": "2026-07-02T02:26:09+09:00", "last_frame_read_age_sec": 30.221, "last_frame_read_at": "2026-07-02T02:26:09+09:00", "mjpeg": {"age_sec": 30.108, "exists": true, "mtime": "2026-07-02T02:26:09.195912+09:00", "mtime_epoch": 1782926769.1959124, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101812}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T02:26:39+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1054.961, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3280}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4323.309, "limit_mb": 8796093022207.996}, "detail": {"pid": 33188, "restart_count": 45, "start_count": 46}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 45, "h264_decode_error_count": 46, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:26:39+09:00", "last_error_summary": "[h264 @ 0x55e252358cc0] error while decoding MB 25 24, bytestream -5", "last_exit_at": "2026-07-01T23:56:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T02:26:41+09:00", "pid": 33188, "process": {"exists": true, "fd_count": 5, "hwm_mb": 130.188, "name": "ffmpeg", "pid": 33188, "rss_mb": 130.188, "state": "S (sleeping)", "threads": 19}, "restart_count": 45, "restart_count_per_hour": 1, "rtsp_timeout_count": 80, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 46}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.52, "sample_count": 2600, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 393700, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:26:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 34.637, "last_frame_pushed_at": "2026-07-02T02:26:09+09:00", "last_frame_read_age_sec": 34.732, "last_frame_read_at": "2026-07-02T02:26:09+09:00", "mjpeg": {"age_sec": 34.62, "exists": true, "mtime": "2026-07-02T02:26:09.195912+09:00", "mtime_epoch": 1782926769.1959124, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101812}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T02:26:43+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1054.961, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3296}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4233.285, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5581248436c0] error while decoding MB 93 22, bytestream -7 [in#0/rtsp @ 0x558124036e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558124036e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 46, "h264_decode_error_count": 47, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:30:31+09:00", "last_error_summary": "[h264 @ 0x5581248436c0] error while decoding MB 93 22, bytestream -7 [in#0/rtsp @ 0x558124036e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558124036e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-01T23:56:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 229.316, "last_started_at": "2026-07-02T02:26:41+09:00", "pid": 33188, "process": {"exists": false, "pid": 33188}, "restart_count": 45, "restart_count_per_hour": 1, "rtsp_timeout_count": 82, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 46}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 100.304, "sample_count": 2215, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 395539, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:30:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.074, "last_frame_pushed_at": "2026-07-02T02:30:31+09:00", "last_frame_read_age_sec": 0.062, "last_frame_read_at": "2026-07-02T02:30:31+09:00", "mjpeg": {"age_sec": 0.057, "exists": true, "mtime": "2026-07-02T02:30:31.073767+09:00", "mtime_epoch": 1782927031.0737672, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 103955}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T02:30:31+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1055.793, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.714, "pushed": 0.714, "read": 0.769}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.415, "count": 13, "max_ms": 3.477, "p95_ms": 3.477}, "gstreamer_push": {"avg_ms": 1.662, "count": 13, "max_ms": 3.828, "p95_ms": 3.828}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.749, "count": 13, "max_ms": 64.982, "p95_ms": 64.982}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.679, "count": 13, "max_ms": 87.952, "p95_ms": 87.952}, "yolo11_640_fire_11class": {"avg_ms": 73.837, "count": 13, "max_ms": 77.825, "p95_ms": 77.825}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.871, "count": 13, "max_ms": 89.862, "p95_ms": 89.862}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.927, "count": 13, "max_ms": 101.895, "p95_ms": 101.895}, "read_interval": {"avg_ms": 2510.489, "count": 14, "max_ms": 17003.064, "p95_ms": 17003.064}, "triton_wait": {"avg_ms": 84.502, "count": 13, "max_ms": 89.875, "p95_ms": 89.875}, "visualize": {"avg_ms": 3.639, "count": 13, "max_ms": 3.728, "p95_ms": 3.728}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.206, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3296}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4232.938, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T02:30:31+09:00", "last_error_summary": "[h264 @ 0x5581248436c0] error while decoding MB 93 22, bytestream -7 [in#0/rtsp @ 0x558124036e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558124036e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 46, "h264_decode_error_count": 47, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:30:31+09:00", "last_error_summary": "[h264 @ 0x5581248436c0] error while decoding MB 93 22, bytestream -7 [in#0/rtsp @ 0x558124036e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558124036e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T02:30:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 229.364, "last_started_at": "2026-07-02T02:26:41+09:00", "pid": 33188, "process": {"exists": false, "pid": 33188}, "restart_count": 45, "restart_count_per_hour": 1, "rtsp_timeout_count": 82, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 46}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 100.336, "sample_count": 2215, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 395540, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:30:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T02:30:31+09:00", "last_frame_read_age_sec": 0.109, "last_frame_read_at": "2026-07-02T02:30:31+09:00", "mjpeg": {"age_sec": 0.104, "exists": true, "mtime": "2026-07-02T02:30:31.073767+09:00", "mtime_epoch": 1782927031.0737672, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 103955}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T02:30:31+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1055.793, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.767, "pushed": 0.767, "read": 0.767}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.406, "count": 14, "max_ms": 3.477, "p95_ms": 3.477}, "gstreamer_push": {"avg_ms": 1.632, "count": 14, "max_ms": 3.828, "p95_ms": 3.828}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.432, "count": 14, "max_ms": 66.312, "p95_ms": 66.312}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.653, "count": 14, "max_ms": 87.952, "p95_ms": 87.952}, "yolo11_640_fire_11class": {"avg_ms": 74.22, "count": 14, "max_ms": 79.191, "p95_ms": 79.191}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.372, "count": 14, "max_ms": 89.891, "p95_ms": 89.891}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.299, "count": 14, "max_ms": 102.138, "p95_ms": 102.138}, "read_interval": {"avg_ms": 2510.489, "count": 14, "max_ms": 17003.064, "p95_ms": 17003.064}, "triton_wait": {"avg_ms": 84.888, "count": 14, "max_ms": 89.903, "p95_ms": 89.903}, "visualize": {"avg_ms": 3.641, "count": 14, "max_ms": 3.728, "p95_ms": 3.728}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.253, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3296}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4232.938, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 46, "h264_decode_error_count": 47, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:30:31+09:00", "last_error_summary": "[h264 @ 0x5581248436c0] error while decoding MB 93 22, bytestream -7 [in#0/rtsp @ 0x558124036e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558124036e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T02:30:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 229.371, "last_started_at": "2026-07-02T02:26:41+09:00", "pid": 33188, "process": {"exists": false, "pid": 33188}, "restart_count": 45, "restart_count_per_hour": 1, "rtsp_timeout_count": 82, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 46}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 100.336, "sample_count": 2215, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 395540, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:30:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-02T02:30:31+09:00", "last_frame_read_age_sec": 0.116, "last_frame_read_at": "2026-07-02T02:30:31+09:00", "mjpeg": {"age_sec": 0.111, "exists": true, "mtime": "2026-07-02T02:30:31.073767+09:00", "mtime_epoch": 1782927031.0737672, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 103955}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T02:30:31+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1055.793, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.767, "pushed": 0.767, "read": 0.767}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.406, "count": 14, "max_ms": 3.477, "p95_ms": 3.477}, "gstreamer_push": {"avg_ms": 1.632, "count": 14, "max_ms": 3.828, "p95_ms": 3.828}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.432, "count": 14, "max_ms": 66.312, "p95_ms": 66.312}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.653, "count": 14, "max_ms": 87.952, "p95_ms": 87.952}, "yolo11_640_fire_11class": {"avg_ms": 74.22, "count": 14, "max_ms": 79.191, "p95_ms": 79.191}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.372, "count": 14, "max_ms": 89.891, "p95_ms": 89.891}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.299, "count": 14, "max_ms": 102.138, "p95_ms": 102.138}, "read_interval": {"avg_ms": 2510.489, "count": 14, "max_ms": 17003.064, "p95_ms": 17003.064}, "triton_wait": {"avg_ms": 84.888, "count": 14, "max_ms": 89.903, "p95_ms": 89.903}, "visualize": {"avg_ms": 3.641, "count": 14, "max_ms": 3.728, "p95_ms": 3.728}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.26, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3296}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4356.73, "limit_mb": 8796093022207.996}, "detail": {"pid": 33340, "restart_count": 46, "start_count": 47}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 46, "h264_decode_error_count": 47, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:30:31+09:00", "last_error_summary": "[h264 @ 0x5581248436c0] error while decoding MB 93 22, bytestream -7 [in#0/rtsp @ 0x558124036e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558124036e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T02:30:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T02:30:33+09:00", "pid": 33340, "process": {"exists": true, "fd_count": 5, "hwm_mb": 156.621, "name": "ffmpeg", "pid": 33340, "rss_mb": 156.621, "state": "R (running)", "threads": 49}, "restart_count": 46, "restart_count_per_hour": 2, "rtsp_timeout_count": 82, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 47}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 100.304, "sample_count": 2173, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 395540, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:30:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.748, "last_frame_pushed_at": "2026-07-02T02:30:31+09:00", "last_frame_read_age_sec": 4.852, "last_frame_read_at": "2026-07-02T02:30:31+09:00", "mjpeg": {"age_sec": 4.731, "exists": true, "mtime": "2026-07-02T02:30:31.189770+09:00", "mtime_epoch": 1782927031.1897695, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 99042}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T02:30:35+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1055.793, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.609, "pushed": 0.609, "read": 0.609}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.406, "count": 14, "max_ms": 3.477, "p95_ms": 3.477}, "gstreamer_push": {"avg_ms": 1.632, "count": 14, "max_ms": 3.828, "p95_ms": 3.828}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.432, "count": 14, "max_ms": 66.312, "p95_ms": 66.312}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.653, "count": 14, "max_ms": 87.952, "p95_ms": 87.952}, "yolo11_640_fire_11class": {"avg_ms": 74.22, "count": 14, "max_ms": 79.191, "p95_ms": 79.191}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.372, "count": 14, "max_ms": 89.891, "p95_ms": 89.891}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.299, "count": 14, "max_ms": 102.138, "p95_ms": 102.138}, "read_interval": {"avg_ms": 2510.489, "count": 14, "max_ms": 17003.064, "p95_ms": 17003.064}, "triton_wait": {"avg_ms": 84.888, "count": 14, "max_ms": 89.903, "p95_ms": 89.903}, "visualize": {"avg_ms": 3.641, "count": 14, "max_ms": 3.728, "p95_ms": 3.728}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.996, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2315}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4283.559, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 6, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5653d124d600] top block unavailable for requested intra mode -1 [h264 @ 0x5653d124d600] error while decoding MB 37 0, bytestream 3827 [h264 @ 0x5653d1d2e6c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5653d1d2e6c0] error while decoding MB 0 27, bytestream 8646 [h264 @ 0x5653d115d980] error while decoding MB 65 11, bytestream -9 [h264 @ 0x5653d115ab80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5653d115ab80] error while decoding MB 0 27, bytestr", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 197, "h264_decode_error_count": 344, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:36:20+09:00", "last_error_summary": "[h264 @ 0x5653d124d600] top block unavailable for requested intra mode -1 [h264 @ 0x5653d124d600] error while decoding MB 37 0, bytestream 3827 [h264 @ 0x5653d1d2e6c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5653d1d2e6c0] error while decoding MB 0 27, bytestream 8646 [h264 @ 0x5653d115d980] error while decoding MB 65 11, bytestream -9 [h264 @ 0x5653d115ab80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5653d115ab80] error while decoding MB 0 27, bytestr", "last_exit_at": "2026-07-02T02:22:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 815.968, "last_started_at": "2026-07-02T02:22:44+09:00", "pid": 33081, "process": {"exists": false, "pid": 33081}, "restart_count": 222, "restart_count_per_hour": 9, "rtsp_timeout_count": 78, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 223}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 100.474, "sample_count": 2673, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 277856, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:36:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.07, "last_frame_pushed_at": "2026-07-02T02:36:20+09:00", "last_frame_read_age_sec": 0.057, "last_frame_read_at": "2026-07-02T02:36:20+09:00", "mjpeg": {"age_sec": 0.055, "exists": true, "mtime": "2026-07-02T02:36:20.060987+09:00", "mtime_epoch": 1782927380.0609875, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 136988}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T02:36:20+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1000.457, "name": "celery", "pid": 237, "rss_mb": 992.406, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 99, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.305, "pushed": 3.305, "read": 3.305}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.575, "count": 99, "max_ms": 2.075, "p95_ms": 1.818}, "gstreamer_push": {"avg_ms": 1.527, "count": 99, "max_ms": 6.37, "p95_ms": 1.542}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.655, "count": 99, "max_ms": 83.431, "p95_ms": 78.791}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.735, "count": 99, "max_ms": 90.614, "p95_ms": 85.962}, "yolo11_640_fire_11class": {"avg_ms": 73.569, "count": 99, "max_ms": 84.788, "p95_ms": 78.801}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.511, "count": 99, "max_ms": 92.186, "p95_ms": 89.055}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.307, "count": 99, "max_ms": 102.715, "p95_ms": 99.539}, "read_interval": {"avg_ms": 303.058, "count": 99, "max_ms": 19151.657, "p95_ms": 115.927}, "triton_wait": {"avg_ms": 83.92, "count": 99, "max_ms": 92.198, "p95_ms": 89.067}, "visualize": {"avg_ms": 3.685, "count": 99, "max_ms": 4.345, "p95_ms": 3.907}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.958, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2315}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4283.031, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T02:36:20+09:00", "last_error_summary": "[h264 @ 0x5653d124d600] top block unavailable for requested intra mode -1 [h264 @ 0x5653d124d600] error while decoding MB 37 0, bytestream 3827 [h264 @ 0x5653d1d2e6c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5653d1d2e6c0] error while decoding MB 0 27, bytestream 8646 [h264 @ 0x5653d115d980] error while decoding MB 65 11, bytestream -9 [h264 @ 0x5653d115ab80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5653d115ab80] error while decoding MB 0 27, bytestr", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 197, "h264_decode_error_count": 344, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:36:20+09:00", "last_error_summary": "[h264 @ 0x5653d124d600] top block unavailable for requested intra mode -1 [h264 @ 0x5653d124d600] error while decoding MB 37 0, bytestream 3827 [h264 @ 0x5653d1d2e6c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5653d1d2e6c0] error while decoding MB 0 27, bytestream 8646 [h264 @ 0x5653d115d980] error while decoding MB 65 11, bytestream -9 [h264 @ 0x5653d115ab80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5653d115ab80] error while decoding MB 0 27, bytestr", "last_exit_at": "2026-07-02T02:36:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 816.015, "last_started_at": "2026-07-02T02:22:44+09:00", "pid": 33081, "process": {"exists": false, "pid": 33081}, "restart_count": 222, "restart_count_per_hour": 9, "rtsp_timeout_count": 78, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 223}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 100.474, "sample_count": 2674, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 277857, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:36:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-02T02:36:20+09:00", "last_frame_read_age_sec": 0.104, "last_frame_read_at": "2026-07-02T02:36:20+09:00", "mjpeg": {"age_sec": 0.101, "exists": true, "mtime": "2026-07-02T02:36:20.060987+09:00", "mtime_epoch": 1782927380.0609875, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 136988}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T02:36:20+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1000.457, "name": "celery", "pid": 237, "rss_mb": 992.406, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 99, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.301, "pushed": 3.301, "read": 3.301}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.574, "count": 99, "max_ms": 2.075, "p95_ms": 1.818}, "gstreamer_push": {"avg_ms": 1.527, "count": 99, "max_ms": 6.37, "p95_ms": 1.542}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.379, "count": 99, "max_ms": 80.155, "p95_ms": 78.093}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.589, "count": 99, "max_ms": 88.328, "p95_ms": 85.474}, "yolo11_640_fire_11class": {"avg_ms": 73.473, "count": 99, "max_ms": 80.168, "p95_ms": 78.607}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.404, "count": 99, "max_ms": 89.948, "p95_ms": 88.97}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.201, "count": 99, "max_ms": 100.216, "p95_ms": 99.16}, "read_interval": {"avg_ms": 303.058, "count": 99, "max_ms": 19151.657, "p95_ms": 115.927}, "triton_wait": {"avg_ms": 83.813, "count": 99, "max_ms": 89.961, "p95_ms": 88.982}, "visualize": {"avg_ms": 3.685, "count": 99, "max_ms": 4.345, "p95_ms": 3.903}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.988, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2315}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4282.906, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 197, "h264_decode_error_count": 344, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:36:20+09:00", "last_error_summary": "[h264 @ 0x5653d124d600] top block unavailable for requested intra mode -1 [h264 @ 0x5653d124d600] error while decoding MB 37 0, bytestream 3827 [h264 @ 0x5653d1d2e6c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5653d1d2e6c0] error while decoding MB 0 27, bytestream 8646 [h264 @ 0x5653d115d980] error while decoding MB 65 11, bytestream -9 [h264 @ 0x5653d115ab80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5653d115ab80] error while decoding MB 0 27, bytestr", "last_exit_at": "2026-07-02T02:36:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 816.022, "last_started_at": "2026-07-02T02:22:44+09:00", "pid": 33081, "process": {"exists": false, "pid": 33081}, "restart_count": 222, "restart_count_per_hour": 9, "rtsp_timeout_count": 78, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 223}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 100.474, "sample_count": 2674, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 277857, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:36:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.017, "last_frame_pushed_at": "2026-07-02T02:36:20+09:00", "last_frame_read_age_sec": 0.112, "last_frame_read_at": "2026-07-02T02:36:20+09:00", "mjpeg": {"age_sec": 0.109, "exists": true, "mtime": "2026-07-02T02:36:20.060987+09:00", "mtime_epoch": 1782927380.0609875, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 136988}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T02:36:20+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1000.457, "name": "celery", "pid": 237, "rss_mb": 992.406, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 99, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.301, "pushed": 3.301, "read": 3.301}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.574, "count": 99, "max_ms": 2.075, "p95_ms": 1.818}, "gstreamer_push": {"avg_ms": 1.527, "count": 99, "max_ms": 6.37, "p95_ms": 1.542}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.379, "count": 99, "max_ms": 80.155, "p95_ms": 78.093}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.589, "count": 99, "max_ms": 88.328, "p95_ms": 85.474}, "yolo11_640_fire_11class": {"avg_ms": 73.473, "count": 99, "max_ms": 80.168, "p95_ms": 78.607}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.404, "count": 99, "max_ms": 89.948, "p95_ms": 88.97}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.201, "count": 99, "max_ms": 100.216, "p95_ms": 99.16}, "read_interval": {"avg_ms": 303.058, "count": 99, "max_ms": 19151.657, "p95_ms": 115.927}, "triton_wait": {"avg_ms": 83.813, "count": 99, "max_ms": 89.961, "p95_ms": 88.982}, "visualize": {"avg_ms": 3.685, "count": 99, "max_ms": 4.345, "p95_ms": 3.903}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2315}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4390.609, "limit_mb": 8796093022207.996}, "detail": {"pid": 33492, "restart_count": 223, "start_count": 224}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 197, "h264_decode_error_count": 344, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:36:20+09:00", "last_error_summary": "[h264 @ 0x5653d124d600] top block unavailable for requested intra mode -1 [h264 @ 0x5653d124d600] error while decoding MB 37 0, bytestream 3827 [h264 @ 0x5653d1d2e6c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5653d1d2e6c0] error while decoding MB 0 27, bytestream 8646 [h264 @ 0x5653d115d980] error while decoding MB 65 11, bytestream -9 [h264 @ 0x5653d115ab80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5653d115ab80] error while decoding MB 0 27, bytestr", "last_exit_at": "2026-07-02T02:36:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T02:36:22+09:00", "pid": 33492, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.363, "name": "ffmpeg", "pid": 33492, "rss_mb": 138.363, "state": "S (sleeping)", "threads": 106}, "restart_count": 223, "restart_count_per_hour": 10, "rtsp_timeout_count": 78, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 224}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 100.483, "sample_count": 2636, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 277857, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:36:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.265, "last_frame_pushed_at": "2026-07-02T02:36:20+09:00", "last_frame_read_age_sec": 4.359, "last_frame_read_at": "2026-07-02T02:36:20+09:00", "mjpeg": {"age_sec": 4.249, "exists": true, "mtime": "2026-07-02T02:36:20.168989+09:00", "mtime_epoch": 1782927380.1689894, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 137177}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T02:36:24+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1000.457, "name": "celery", "pid": 237, "rss_mb": 992.406, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 61, "pushed_frames": 61, "read_frames": 60, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.039, "pushed": 2.039, "read": 2.005}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.549, "count": 61, "max_ms": 2.002, "p95_ms": 1.645}, "gstreamer_push": {"avg_ms": 1.608, "count": 61, "max_ms": 6.37, "p95_ms": 3.818}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.825, "count": 61, "max_ms": 80.155, "p95_ms": 78.791}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.701, "count": 61, "max_ms": 85.962, "p95_ms": 84.078}, "yolo11_640_fire_11class": {"avg_ms": 73.605, "count": 61, "max_ms": 80.168, "p95_ms": 78.607}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.045, "count": 61, "max_ms": 89.948, "p95_ms": 87.52}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.794, "count": 61, "max_ms": 100.216, "p95_ms": 98.075}, "read_interval": {"avg_ms": 427.652, "count": 60, "max_ms": 19151.657, "p95_ms": 114.781}, "triton_wait": {"avg_ms": 83.406, "count": 61, "max_ms": 89.961, "p95_ms": 87.532}, "visualize": {"avg_ms": 3.67, "count": 61, "max_ms": 3.963, "p95_ms": 3.782}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.922, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3345}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4239.027, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55975e96f9c0] error while decoding MB 16 42, bytestream -5 [in#0/rtsp @ 0x55975dc89e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55975dc89e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 47, "h264_decode_error_count": 48, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:41:15+09:00", "last_error_summary": "[h264 @ 0x55975e96f9c0] error while decoding MB 16 42, bytestream -5 [in#0/rtsp @ 0x55975dc89e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55975dc89e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T02:30:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 641.756, "last_started_at": "2026-07-02T02:30:33+09:00", "pid": 33340, "process": {"exists": false, "pid": 33340}, "restart_count": 46, "restart_count_per_hour": 2, "rtsp_timeout_count": 84, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 47}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.107, "sample_count": 2632, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 401498, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:41:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.075, "last_frame_pushed_at": "2026-07-02T02:41:15+09:00", "last_frame_read_age_sec": 0.062, "last_frame_read_at": "2026-07-02T02:41:15+09:00", "mjpeg": {"age_sec": 0.052, "exists": true, "mtime": "2026-07-02T02:41:15.622813+09:00", "mtime_epoch": 1782927675.6228132, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 103736}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T02:41:15+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1056.051, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 73, "pushed_frames": 73, "read_frames": 73, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.435, "pushed": 2.435, "read": 2.435}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.493, "count": 73, "max_ms": 4.902, "p95_ms": 4.562}, "gstreamer_push": {"avg_ms": 1.506, "count": 73, "max_ms": 6.13, "p95_ms": 3.582}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.122, "count": 73, "max_ms": 77.642, "p95_ms": 74.543}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.89, "count": 73, "max_ms": 87.554, "p95_ms": 84.589}, "yolo11_640_fire_11class": {"avg_ms": 65.147, "count": 73, "max_ms": 77.629, "p95_ms": 74.985}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.282, "count": 73, "max_ms": 89.226, "p95_ms": 87.394}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.506, "count": 73, "max_ms": 101.276, "p95_ms": 99.608}, "read_interval": {"avg_ms": 411.228, "count": 73, "max_ms": 13672.562, "p95_ms": 119.198}, "triton_wait": {"avg_ms": 73.15, "count": 73, "max_ms": 89.238, "p95_ms": 87.406}, "visualize": {"avg_ms": 3.685, "count": 73, "max_ms": 4.181, "p95_ms": 3.947}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.985, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3345}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4238.496, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T02:41:15+09:00", "last_error_summary": "[h264 @ 0x55975e96f9c0] error while decoding MB 16 42, bytestream -5 [in#0/rtsp @ 0x55975dc89e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55975dc89e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 47, "h264_decode_error_count": 48, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:41:15+09:00", "last_error_summary": "[h264 @ 0x55975e96f9c0] error while decoding MB 16 42, bytestream -5 [in#0/rtsp @ 0x55975dc89e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55975dc89e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T02:41:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 641.798, "last_started_at": "2026-07-02T02:30:33+09:00", "pid": 33340, "process": {"exists": false, "pid": 33340}, "restart_count": 46, "restart_count_per_hour": 2, "rtsp_timeout_count": 84, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 47}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.107, "sample_count": 2633, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 401499, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:41:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T02:41:15+09:00", "last_frame_read_age_sec": 0.104, "last_frame_read_at": "2026-07-02T02:41:15+09:00", "mjpeg": {"age_sec": 0.095, "exists": true, "mtime": "2026-07-02T02:41:15.622813+09:00", "mtime_epoch": 1782927675.6228132, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 103736}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T02:41:15+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1056.051, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 73, "pushed_frames": 73, "read_frames": 72, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.44, "pushed": 2.44, "read": 2.407}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.492, "count": 73, "max_ms": 4.902, "p95_ms": 4.562}, "gstreamer_push": {"avg_ms": 1.506, "count": 73, "max_ms": 6.13, "p95_ms": 3.582}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.018, "count": 73, "max_ms": 77.642, "p95_ms": 74.543}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.941, "count": 73, "max_ms": 87.554, "p95_ms": 84.589}, "yolo11_640_fire_11class": {"avg_ms": 65.131, "count": 73, "max_ms": 77.629, "p95_ms": 74.985}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.269, "count": 73, "max_ms": 89.226, "p95_ms": 87.394}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.497, "count": 73, "max_ms": 101.276, "p95_ms": 99.608}, "read_interval": {"avg_ms": 415.366, "count": 72, "max_ms": 13672.562, "p95_ms": 119.198}, "triton_wait": {"avg_ms": 73.137, "count": 73, "max_ms": 89.238, "p95_ms": 87.406}, "visualize": {"avg_ms": 3.686, "count": 73, "max_ms": 4.181, "p95_ms": 3.947}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.917, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3345}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4238.523, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 47, "h264_decode_error_count": 48, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:41:15+09:00", "last_error_summary": "[h264 @ 0x55975e96f9c0] error while decoding MB 16 42, bytestream -5 [in#0/rtsp @ 0x55975dc89e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55975dc89e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T02:41:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 641.806, "last_started_at": "2026-07-02T02:30:33+09:00", "pid": 33340, "process": {"exists": false, "pid": 33340}, "restart_count": 46, "restart_count_per_hour": 2, "rtsp_timeout_count": 84, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 47}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.107, "sample_count": 2633, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 401499, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:41:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-02T02:41:15+09:00", "last_frame_read_age_sec": 0.112, "last_frame_read_at": "2026-07-02T02:41:15+09:00", "mjpeg": {"age_sec": 0.103, "exists": true, "mtime": "2026-07-02T02:41:15.622813+09:00", "mtime_epoch": 1782927675.6228132, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 103736}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T02:41:15+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1056.051, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 73, "pushed_frames": 73, "read_frames": 72, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.439, "pushed": 2.439, "read": 2.406}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.492, "count": 73, "max_ms": 4.902, "p95_ms": 4.562}, "gstreamer_push": {"avg_ms": 1.506, "count": 73, "max_ms": 6.13, "p95_ms": 3.582}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.018, "count": 73, "max_ms": 77.642, "p95_ms": 74.543}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.941, "count": 73, "max_ms": 87.554, "p95_ms": 84.589}, "yolo11_640_fire_11class": {"avg_ms": 65.131, "count": 73, "max_ms": 77.629, "p95_ms": 74.985}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.269, "count": 73, "max_ms": 89.226, "p95_ms": 87.394}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.497, "count": 73, "max_ms": 101.276, "p95_ms": 99.608}, "read_interval": {"avg_ms": 415.366, "count": 72, "max_ms": 13672.562, "p95_ms": 119.198}, "triton_wait": {"avg_ms": 73.137, "count": 73, "max_ms": 89.238, "p95_ms": 87.406}, "visualize": {"avg_ms": 3.686, "count": 73, "max_ms": 4.181, "p95_ms": 3.947}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.925, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3345}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4258.789, "limit_mb": 8796093022207.996}, "detail": {"pid": 33599, "restart_count": 47, "start_count": 48}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 47, "h264_decode_error_count": 48, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:41:15+09:00", "last_error_summary": "[h264 @ 0x55975e96f9c0] error while decoding MB 16 42, bytestream -5 [in#0/rtsp @ 0x55975dc89e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55975dc89e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T02:41:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T02:41:18+09:00", "pid": 33599, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.516, "name": "ffmpeg", "pid": 33599, "rss_mb": 57.516, "state": "S (sleeping)", "threads": 1}, "restart_count": 47, "restart_count_per_hour": 3, "rtsp_timeout_count": 84, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 48}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.167, "sample_count": 2575, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 401499, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:41:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.92, "last_frame_pushed_at": "2026-07-02T02:41:15+09:00", "last_frame_read_age_sec": 5.018, "last_frame_read_at": "2026-07-02T02:41:15+09:00", "mjpeg": {"age_sec": 4.905, "exists": true, "mtime": "2026-07-02T02:41:15.726815+09:00", "mtime_epoch": 1782927675.7268152, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 99947}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T02:41:20+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1056.051, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 24, "pushed_frames": 24, "read_frames": 23, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.802, "pushed": 0.802, "read": 0.768}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.632, "count": 24, "max_ms": 4.902, "p95_ms": 4.879}, "gstreamer_push": {"avg_ms": 1.431, "count": 24, "max_ms": 3.582, "p95_ms": 1.427}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.953, "count": 24, "max_ms": 77.642, "p95_ms": 70.81}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.39, "count": 24, "max_ms": 87.554, "p95_ms": 84.992}, "yolo11_640_fire_11class": {"avg_ms": 70.358, "count": 24, "max_ms": 77.629, "p95_ms": 74.549}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.885, "count": 24, "max_ms": 89.226, "p95_ms": 88.232}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.506, "count": 24, "max_ms": 101.276, "p95_ms": 100.443}, "read_interval": {"avg_ms": 1087.66, "count": 23, "max_ms": 13672.562, "p95_ms": 9008.126}, "triton_wait": {"avg_ms": 80.938, "count": 24, "max_ms": 89.238, "p95_ms": 88.245}, "visualize": {"avg_ms": 3.737, "count": 24, "max_ms": 4.181, "p95_ms": 4.177}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.942, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2349}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4284.773, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x560f1c1baa00] top block unavailable for requested intra mode -1 [h264 @ 0x560f1c1baa00] error while decoding MB 74 0, bytestream 58292 [in#0/rtsp @ 0x560f1b3f5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x560f1b3f5e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 198, "h264_decode_error_count": 345, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:43:50+09:00", "last_error_summary": "[h264 @ 0x560f1c1baa00] top block unavailable for requested intra mode -1 [h264 @ 0x560f1c1baa00] error while decoding MB 74 0, bytestream 58292 [in#0/rtsp @ 0x560f1b3f5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x560f1b3f5e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T02:36:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 448.323, "last_started_at": "2026-07-02T02:36:22+09:00", "pid": 33492, "process": {"exists": false, "pid": 33492}, "restart_count": 223, "restart_count_per_hour": 8, "rtsp_timeout_count": 80, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 224}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.73, "sample_count": 2719, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 281935, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:43:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.07, "last_frame_pushed_at": "2026-07-02T02:43:50+09:00", "last_frame_read_age_sec": 0.058, "last_frame_read_at": "2026-07-02T02:43:50+09:00", "mjpeg": {"age_sec": 0.054, "exists": true, "mtime": "2026-07-02T02:43:50.685808+09:00", "mtime_epoch": 1782927830.6858077, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 124615}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T02:43:50+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1001.59, "name": "celery", "pid": 237, "rss_mb": 994.105, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 99, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.305, "pushed": 3.305, "read": 3.305}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.54, "count": 99, "max_ms": 2.155, "p95_ms": 1.57}, "gstreamer_push": {"avg_ms": 1.526, "count": 99, "max_ms": 6.449, "p95_ms": 1.544}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.201, "count": 99, "max_ms": 77.956, "p95_ms": 76.003}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.061, "count": 99, "max_ms": 87.091, "p95_ms": 84.539}, "yolo11_640_fire_11class": {"avg_ms": 71.512, "count": 99, "max_ms": 77.003, "p95_ms": 76.19}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.889, "count": 99, "max_ms": 88.719, "p95_ms": 87.699}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.745, "count": 99, "max_ms": 99.73, "p95_ms": 98.005}, "read_interval": {"avg_ms": 302.97, "count": 99, "max_ms": 10131.233, "p95_ms": 117.436}, "triton_wait": {"avg_ms": 81.191, "count": 99, "max_ms": 88.732, "p95_ms": 87.712}, "visualize": {"avg_ms": 3.714, "count": 99, "max_ms": 4.37, "p95_ms": 3.929}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.954, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2349}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4284.25, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T02:43:50+09:00", "last_error_summary": "[h264 @ 0x560f1c1baa00] top block unavailable for requested intra mode -1 [h264 @ 0x560f1c1baa00] error while decoding MB 74 0, bytestream 58292 [in#0/rtsp @ 0x560f1b3f5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x560f1b3f5e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 198, "h264_decode_error_count": 345, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:43:50+09:00", "last_error_summary": "[h264 @ 0x560f1c1baa00] top block unavailable for requested intra mode -1 [h264 @ 0x560f1c1baa00] error while decoding MB 74 0, bytestream 58292 [in#0/rtsp @ 0x560f1b3f5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x560f1b3f5e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T02:43:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 448.348, "last_started_at": "2026-07-02T02:36:22+09:00", "pid": 33492, "process": {"exists": false, "pid": 33492}, "restart_count": 223, "restart_count_per_hour": 8, "rtsp_timeout_count": 80, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 224}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.73, "sample_count": 2720, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 281936, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:43:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T02:43:50+09:00", "last_frame_read_age_sec": 0.083, "last_frame_read_at": "2026-07-02T02:43:50+09:00", "mjpeg": {"age_sec": 0.079, "exists": true, "mtime": "2026-07-02T02:43:50.685808+09:00", "mtime_epoch": 1782927830.6858077, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 124615}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T02:43:50+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1001.59, "name": "celery", "pid": 237, "rss_mb": 994.105, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.336, "pushed": 3.336, "read": 3.302}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.543, "count": 100, "max_ms": 2.155, "p95_ms": 1.57}, "gstreamer_push": {"avg_ms": 1.524, "count": 100, "max_ms": 6.449, "p95_ms": 1.535}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.016, "count": 100, "max_ms": 77.956, "p95_ms": 75.981}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.834, "count": 100, "max_ms": 87.091, "p95_ms": 84.067}, "yolo11_640_fire_11class": {"avg_ms": 71.312, "count": 100, "max_ms": 77.003, "p95_ms": 75.969}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.697, "count": 100, "max_ms": 88.719, "p95_ms": 87.546}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.575, "count": 100, "max_ms": 99.73, "p95_ms": 97.961}, "read_interval": {"avg_ms": 302.97, "count": 99, "max_ms": 10131.233, "p95_ms": 117.436}, "triton_wait": {"avg_ms": 80.995, "count": 100, "max_ms": 88.732, "p95_ms": 87.558}, "visualize": {"avg_ms": 3.738, "count": 100, "max_ms": 6.092, "p95_ms": 3.929}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.979, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2349}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4284.289, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 198, "h264_decode_error_count": 345, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:43:50+09:00", "last_error_summary": "[h264 @ 0x560f1c1baa00] top block unavailable for requested intra mode -1 [h264 @ 0x560f1c1baa00] error while decoding MB 74 0, bytestream 58292 [in#0/rtsp @ 0x560f1b3f5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x560f1b3f5e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T02:43:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 448.357, "last_started_at": "2026-07-02T02:36:22+09:00", "pid": 33492, "process": {"exists": false, "pid": 33492}, "restart_count": 223, "restart_count_per_hour": 8, "rtsp_timeout_count": 80, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 224}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.73, "sample_count": 2720, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 281936, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:43:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-02T02:43:50+09:00", "last_frame_read_age_sec": 0.092, "last_frame_read_at": "2026-07-02T02:43:50+09:00", "mjpeg": {"age_sec": 0.088, "exists": true, "mtime": "2026-07-02T02:43:50.685808+09:00", "mtime_epoch": 1782927830.6858077, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 124615}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T02:43:50+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1001.59, "name": "celery", "pid": 237, "rss_mb": 994.105, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.335, "pushed": 3.335, "read": 3.301}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.543, "count": 100, "max_ms": 2.155, "p95_ms": 1.57}, "gstreamer_push": {"avg_ms": 1.524, "count": 100, "max_ms": 6.449, "p95_ms": 1.535}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.016, "count": 100, "max_ms": 77.956, "p95_ms": 75.981}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.834, "count": 100, "max_ms": 87.091, "p95_ms": 84.067}, "yolo11_640_fire_11class": {"avg_ms": 71.312, "count": 100, "max_ms": 77.003, "p95_ms": 75.969}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.697, "count": 100, "max_ms": 88.719, "p95_ms": 87.546}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.575, "count": 100, "max_ms": 99.73, "p95_ms": 97.961}, "read_interval": {"avg_ms": 302.97, "count": 99, "max_ms": 10131.233, "p95_ms": 117.436}, "triton_wait": {"avg_ms": 80.995, "count": 100, "max_ms": 88.732, "p95_ms": 87.558}, "visualize": {"avg_ms": 3.738, "count": 100, "max_ms": 6.092, "p95_ms": 3.929}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.988, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2349}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4391.746, "limit_mb": 8796093022207.996}, "detail": {"pid": 33751, "restart_count": 224, "start_count": 225}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 198, "h264_decode_error_count": 345, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:43:50+09:00", "last_error_summary": "[h264 @ 0x560f1c1baa00] top block unavailable for requested intra mode -1 [h264 @ 0x560f1c1baa00] error while decoding MB 74 0, bytestream 58292 [in#0/rtsp @ 0x560f1b3f5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x560f1b3f5e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T02:43:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T02:43:52+09:00", "pid": 33751, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.227, "name": "ffmpeg", "pid": 33751, "rss_mb": 139.227, "state": "S (sleeping)", "threads": 106}, "restart_count": 224, "restart_count_per_hour": 9, "rtsp_timeout_count": 80, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 225}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.771, "sample_count": 2675, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 281936, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:43:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.054, "last_frame_pushed_at": "2026-07-02T02:43:50+09:00", "last_frame_read_age_sec": 4.131, "last_frame_read_at": "2026-07-02T02:43:50+09:00", "mjpeg": {"age_sec": 4.039, "exists": true, "mtime": "2026-07-02T02:43:50.773809+09:00", "mtime_epoch": 1782927830.7738094, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 123620}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T02:43:54+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1001.59, "name": "celery", "pid": 237, "rss_mb": 994.105, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 63, "pushed_frames": 63, "read_frames": 62, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.103, "pushed": 2.103, "read": 2.069}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.539, "count": 63, "max_ms": 1.902, "p95_ms": 1.57}, "gstreamer_push": {"avg_ms": 1.397, "count": 63, "max_ms": 1.498, "p95_ms": 1.464}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.991, "count": 63, "max_ms": 76.521, "p95_ms": 75.797}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.493, "count": 63, "max_ms": 87.091, "p95_ms": 85.327}, "yolo11_640_fire_11class": {"avg_ms": 70.192, "count": 63, "max_ms": 76.842, "p95_ms": 75.694}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.653, "count": 63, "max_ms": 88.719, "p95_ms": 87.171}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.657, "count": 63, "max_ms": 99.73, "p95_ms": 97.804}, "read_interval": {"avg_ms": 418.149, "count": 62, "max_ms": 10131.233, "p95_ms": 117.285}, "triton_wait": {"avg_ms": 80.12, "count": 63, "max_ms": 88.732, "p95_ms": 87.184}, "visualize": {"avg_ms": 3.74, "count": 63, "max_ms": 6.092, "p95_ms": 3.855}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.962, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3374}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4235.516, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x55f1ba280200] missing picture in access unit with size 18 [h264 @ 0x55f1baa85e00] no frame! [in#0/rtsp @ 0x55f1ba27be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f1ba27be00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55f1ba280840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 48, "h264_decode_error_count": 49, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:48:01+09:00", "last_error_summary": "[NULL @ 0x55f1ba280200] missing picture in access unit with size 18 [h264 @ 0x55f1baa85e00] no frame! [in#0/rtsp @ 0x55f1ba27be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f1ba27be00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55f1ba280840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T02:41:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 402.655, "last_started_at": "2026-07-02T02:41:18+09:00", "pid": 33599, "process": {"exists": false, "pid": 33599}, "restart_count": 47, "restart_count_per_hour": 3, "rtsp_timeout_count": 86, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 48}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.538, "sample_count": 2562, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 404980, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:48:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.079, "last_frame_pushed_at": "2026-07-02T02:48:01+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-02T02:48:01+09:00", "mjpeg": {"age_sec": 0.063, "exists": true, "mtime": "2026-07-02T02:48:01.222589+09:00", "mtime_epoch": 1782928081.222589, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 98650}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T02:48:01+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1057.539, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.69, "pushed": 0.69, "read": 0.748}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.502, "count": 12, "max_ms": 4.546, "p95_ms": 4.546}, "gstreamer_push": {"avg_ms": 1.341, "count": 12, "max_ms": 1.401, "p95_ms": 1.401}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.924, "count": 12, "max_ms": 76.167, "p95_ms": 76.167}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.903, "count": 12, "max_ms": 80.275, "p95_ms": 80.275}, "yolo11_640_fire_11class": {"avg_ms": 72.437, "count": 12, "max_ms": 78.085, "p95_ms": 78.085}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.977, "count": 12, "max_ms": 85.291, "p95_ms": 85.291}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.218, "count": 12, "max_ms": 97.532, "p95_ms": 97.532}, "read_interval": {"avg_ms": 2450.132, "count": 13, "max_ms": 16132.179, "p95_ms": 16132.179}, "triton_wait": {"avg_ms": 79.989, "count": 12, "max_ms": 85.302, "p95_ms": 85.302}, "visualize": {"avg_ms": 3.68, "count": 12, "max_ms": 3.807, "p95_ms": 3.807}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.38, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3374}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4234.695, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T02:48:01+09:00", "last_error_summary": "[NULL @ 0x55f1ba280200] missing picture in access unit with size 18 [h264 @ 0x55f1baa85e00] no frame! [in#0/rtsp @ 0x55f1ba27be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f1ba27be00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55f1ba280840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 48, "h264_decode_error_count": 49, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:48:01+09:00", "last_error_summary": "[NULL @ 0x55f1ba280200] missing picture in access unit with size 18 [h264 @ 0x55f1baa85e00] no frame! [in#0/rtsp @ 0x55f1ba27be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f1ba27be00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55f1ba280840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T02:48:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 402.694, "last_started_at": "2026-07-02T02:41:18+09:00", "pid": 33599, "process": {"exists": false, "pid": 33599}, "restart_count": 47, "restart_count_per_hour": 3, "rtsp_timeout_count": 86, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 48}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.538, "sample_count": 2563, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 404981, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:48:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T02:48:01+09:00", "last_frame_read_age_sec": 0.104, "last_frame_read_at": "2026-07-02T02:48:01+09:00", "mjpeg": {"age_sec": 0.103, "exists": true, "mtime": "2026-07-02T02:48:01.222589+09:00", "mtime_epoch": 1782928081.222589, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 98650}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T02:48:01+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1057.539, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.746, "pushed": 0.746, "read": 0.746}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.489, "count": 13, "max_ms": 4.546, "p95_ms": 4.546}, "gstreamer_push": {"avg_ms": 1.339, "count": 13, "max_ms": 1.401, "p95_ms": 1.401}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.927, "count": 13, "max_ms": 76.167, "p95_ms": 76.167}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.896, "count": 13, "max_ms": 80.275, "p95_ms": 80.275}, "yolo11_640_fire_11class": {"avg_ms": 72.558, "count": 13, "max_ms": 78.085, "p95_ms": 78.085}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.267, "count": 13, "max_ms": 85.291, "p95_ms": 85.291}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.494, "count": 13, "max_ms": 97.532, "p95_ms": 97.532}, "read_interval": {"avg_ms": 2450.132, "count": 13, "max_ms": 16132.179, "p95_ms": 16132.179}, "triton_wait": {"avg_ms": 80.279, "count": 13, "max_ms": 85.302, "p95_ms": 85.302}, "visualize": {"avg_ms": 3.679, "count": 13, "max_ms": 3.807, "p95_ms": 3.807}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.42, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3374}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4234.648, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 48, "h264_decode_error_count": 49, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:48:01+09:00", "last_error_summary": "[NULL @ 0x55f1ba280200] missing picture in access unit with size 18 [h264 @ 0x55f1baa85e00] no frame! [in#0/rtsp @ 0x55f1ba27be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f1ba27be00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55f1ba280840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T02:48:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 402.702, "last_started_at": "2026-07-02T02:41:18+09:00", "pid": 33599, "process": {"exists": false, "pid": 33599}, "restart_count": 47, "restart_count_per_hour": 3, "rtsp_timeout_count": 86, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 48}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.538, "sample_count": 2563, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 404981, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:48:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-02T02:48:01+09:00", "last_frame_read_age_sec": 0.111, "last_frame_read_at": "2026-07-02T02:48:01+09:00", "mjpeg": {"age_sec": 0.11, "exists": true, "mtime": "2026-07-02T02:48:01.222589+09:00", "mtime_epoch": 1782928081.222589, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 98650}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T02:48:01+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1057.539, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.746, "pushed": 0.746, "read": 0.746}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.489, "count": 13, "max_ms": 4.546, "p95_ms": 4.546}, "gstreamer_push": {"avg_ms": 1.339, "count": 13, "max_ms": 1.401, "p95_ms": 1.401}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.927, "count": 13, "max_ms": 76.167, "p95_ms": 76.167}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.896, "count": 13, "max_ms": 80.275, "p95_ms": 80.275}, "yolo11_640_fire_11class": {"avg_ms": 72.558, "count": 13, "max_ms": 78.085, "p95_ms": 78.085}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.267, "count": 13, "max_ms": 85.291, "p95_ms": 85.291}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.494, "count": 13, "max_ms": 97.532, "p95_ms": 97.532}, "read_interval": {"avg_ms": 2450.132, "count": 13, "max_ms": 16132.179, "p95_ms": 16132.179}, "triton_wait": {"avg_ms": 80.279, "count": 13, "max_ms": 85.302, "p95_ms": 85.302}, "visualize": {"avg_ms": 3.679, "count": 13, "max_ms": 3.807, "p95_ms": 3.807}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.427, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3374}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4257.934, "limit_mb": 8796093022207.996}, "detail": {"pid": 33858, "restart_count": 48, "start_count": 49}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 48, "h264_decode_error_count": 49, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:48:01+09:00", "last_error_summary": "[NULL @ 0x55f1ba280200] missing picture in access unit with size 18 [h264 @ 0x55f1baa85e00] no frame! [in#0/rtsp @ 0x55f1ba27be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f1ba27be00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55f1ba280840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T02:48:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T02:48:03+09:00", "pid": 33858, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.133, "name": "ffmpeg", "pid": 33858, "rss_mb": 58.133, "state": "S (sleeping)", "threads": 1}, "restart_count": 48, "restart_count_per_hour": 4, "rtsp_timeout_count": 86, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 49}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.487, "sample_count": 2524, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 404981, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:48:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.378, "last_frame_pushed_at": "2026-07-02T02:48:01+09:00", "last_frame_read_age_sec": 4.476, "last_frame_read_at": "2026-07-02T02:48:01+09:00", "mjpeg": {"age_sec": 4.363, "exists": true, "mtime": "2026-07-02T02:48:01.334591+09:00", "mtime_epoch": 1782928081.3345912, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 100172}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T02:48:05+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1057.539, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.597, "pushed": 0.597, "read": 0.597}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.489, "count": 13, "max_ms": 4.546, "p95_ms": 4.546}, "gstreamer_push": {"avg_ms": 1.339, "count": 13, "max_ms": 1.401, "p95_ms": 1.401}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.927, "count": 13, "max_ms": 76.167, "p95_ms": 76.167}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.896, "count": 13, "max_ms": 80.275, "p95_ms": 80.275}, "yolo11_640_fire_11class": {"avg_ms": 72.558, "count": 13, "max_ms": 78.085, "p95_ms": 78.085}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.267, "count": 13, "max_ms": 85.291, "p95_ms": 85.291}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.494, "count": 13, "max_ms": 97.532, "p95_ms": 97.532}, "read_interval": {"avg_ms": 2450.132, "count": 13, "max_ms": 16132.179, "p95_ms": 16132.179}, "triton_wait": {"avg_ms": 80.279, "count": 13, "max_ms": 85.302, "p95_ms": 85.302}, "visualize": {"avg_ms": 3.679, "count": 13, "max_ms": 3.807, "p95_ms": 3.807}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.793, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3394}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4260.453, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x561b7356c880] error while decoding MB 119 43, bytestream -5 [in#0/rtsp @ 0x561b72bc4e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561b72bc4e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 49, "h264_decode_error_count": 50, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:52:44+09:00", "last_error_summary": "[h264 @ 0x561b7356c880] error while decoding MB 119 43, bytestream -5 [in#0/rtsp @ 0x561b72bc4e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561b72bc4e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T02:48:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 280.93, "last_started_at": "2026-07-02T02:48:03+09:00", "pid": 33858, "process": {"exists": false, "pid": 33858}, "restart_count": 48, "restart_count_per_hour": 4, "rtsp_timeout_count": 88, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 49}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.022, "sample_count": 2366, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 407335, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:52:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-02T02:52:44+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-02T02:52:44+09:00", "mjpeg": {"age_sec": 0.06, "exists": true, "mtime": "2026-07-02T02:52:44.566982+09:00", "mtime_epoch": 1782928364.5669818, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 100186}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T02:52:44+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1064.301, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 65, "pushed_frames": 65, "read_frames": 64, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.17, "pushed": 2.17, "read": 2.137}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.46, "count": 65, "max_ms": 4.597, "p95_ms": 3.549}, "gstreamer_push": {"avg_ms": 1.394, "count": 65, "max_ms": 3.844, "p95_ms": 1.437}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.701, "count": 65, "max_ms": 80.85, "p95_ms": 76.422}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.484, "count": 65, "max_ms": 81.889, "p95_ms": 80.305}, "yolo11_640_fire_11class": {"avg_ms": 68.677, "count": 65, "max_ms": 80.837, "p95_ms": 76.742}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.553, "count": 65, "max_ms": 89.815, "p95_ms": 87.557}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.315, "count": 65, "max_ms": 102.085, "p95_ms": 99.821}, "read_interval": {"avg_ms": 468.434, "count": 64, "max_ms": 23312.03, "p95_ms": 117.947}, "triton_wait": {"avg_ms": 77.049, "count": 65, "max_ms": 89.827, "p95_ms": 87.57}, "visualize": {"avg_ms": 3.671, "count": 65, "max_ms": 4.425, "p95_ms": 3.893}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.951, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3394}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4260.277, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T02:52:44+09:00", "last_error_summary": "[h264 @ 0x561b7356c880] error while decoding MB 119 43, bytestream -5 [in#0/rtsp @ 0x561b72bc4e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561b72bc4e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 49, "h264_decode_error_count": 50, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:52:44+09:00", "last_error_summary": "[h264 @ 0x561b7356c880] error while decoding MB 119 43, bytestream -5 [in#0/rtsp @ 0x561b72bc4e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561b72bc4e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T02:52:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 280.947, "last_started_at": "2026-07-02T02:48:03+09:00", "pid": 33858, "process": {"exists": false, "pid": 33858}, "restart_count": 48, "restart_count_per_hour": 4, "rtsp_timeout_count": 88, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 49}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.022, "sample_count": 2366, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 407335, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:52:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-02T02:52:44+09:00", "last_frame_read_age_sec": 0.082, "last_frame_read_at": "2026-07-02T02:52:44+09:00", "mjpeg": {"age_sec": 0.077, "exists": true, "mtime": "2026-07-02T02:52:44.566982+09:00", "mtime_epoch": 1782928364.5669818, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 100186}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T02:52:44+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1064.301, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 65, "pushed_frames": 65, "read_frames": 64, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.169, "pushed": 2.169, "read": 2.136}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.46, "count": 65, "max_ms": 4.597, "p95_ms": 3.549}, "gstreamer_push": {"avg_ms": 1.394, "count": 65, "max_ms": 3.844, "p95_ms": 1.437}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.701, "count": 65, "max_ms": 80.85, "p95_ms": 76.422}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.484, "count": 65, "max_ms": 81.889, "p95_ms": 80.305}, "yolo11_640_fire_11class": {"avg_ms": 68.677, "count": 65, "max_ms": 80.837, "p95_ms": 76.742}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.553, "count": 65, "max_ms": 89.815, "p95_ms": 87.557}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.315, "count": 65, "max_ms": 102.085, "p95_ms": 99.821}, "read_interval": {"avg_ms": 468.434, "count": 64, "max_ms": 23312.03, "p95_ms": 117.947}, "triton_wait": {"avg_ms": 77.049, "count": 65, "max_ms": 89.827, "p95_ms": 87.57}, "visualize": {"avg_ms": 3.671, "count": 65, "max_ms": 4.425, "p95_ms": 3.893}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.966, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3394}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4260.098, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 49, "h264_decode_error_count": 50, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:52:44+09:00", "last_error_summary": "[h264 @ 0x561b7356c880] error while decoding MB 119 43, bytestream -5 [in#0/rtsp @ 0x561b72bc4e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561b72bc4e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T02:52:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 280.955, "last_started_at": "2026-07-02T02:48:03+09:00", "pid": 33858, "process": {"exists": false, "pid": 33858}, "restart_count": 48, "restart_count_per_hour": 4, "rtsp_timeout_count": 88, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 49}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.022, "sample_count": 2366, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 407335, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:52:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.023, "last_frame_pushed_at": "2026-07-02T02:52:44+09:00", "last_frame_read_age_sec": 0.09, "last_frame_read_at": "2026-07-02T02:52:44+09:00", "mjpeg": {"age_sec": 0.009, "exists": true, "mtime": "2026-07-02T02:52:44.642981+09:00", "mtime_epoch": 1782928364.6429813, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 100928}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T02:52:44+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1064.301, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 65, "pushed_frames": 65, "read_frames": 64, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.169, "pushed": 2.169, "read": 2.135}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.46, "count": 65, "max_ms": 4.597, "p95_ms": 3.549}, "gstreamer_push": {"avg_ms": 1.394, "count": 65, "max_ms": 3.844, "p95_ms": 1.437}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.701, "count": 65, "max_ms": 80.85, "p95_ms": 76.422}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.484, "count": 65, "max_ms": 81.889, "p95_ms": 80.305}, "yolo11_640_fire_11class": {"avg_ms": 68.677, "count": 65, "max_ms": 80.837, "p95_ms": 76.742}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.553, "count": 65, "max_ms": 89.815, "p95_ms": 87.557}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.315, "count": 65, "max_ms": 102.085, "p95_ms": 99.821}, "read_interval": {"avg_ms": 468.434, "count": 64, "max_ms": 23312.03, "p95_ms": 117.947}, "triton_wait": {"avg_ms": 77.049, "count": 65, "max_ms": 89.827, "p95_ms": 87.57}, "visualize": {"avg_ms": 3.671, "count": 65, "max_ms": 4.425, "p95_ms": 3.893}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.974, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3394}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4277.559, "limit_mb": 8796093022207.996}, "detail": {"pid": 34010, "restart_count": 49, "start_count": 50}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 49, "h264_decode_error_count": 50, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:52:44+09:00", "last_error_summary": "[h264 @ 0x561b7356c880] error while decoding MB 119 43, bytestream -5 [in#0/rtsp @ 0x561b72bc4e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561b72bc4e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T02:52:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T02:52:46+09:00", "pid": 34010, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.301, "name": "ffmpeg", "pid": 34010, "rss_mb": 57.301, "state": "S (sleeping)", "threads": 1}, "restart_count": 49, "restart_count_per_hour": 5, "rtsp_timeout_count": 88, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 50}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.022, "sample_count": 2366, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 407335, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:52:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.061, "last_frame_pushed_at": "2026-07-02T02:52:44+09:00", "last_frame_read_age_sec": 4.127, "last_frame_read_at": "2026-07-02T02:52:44+09:00", "mjpeg": {"age_sec": 4.046, "exists": true, "mtime": "2026-07-02T02:52:44.642981+09:00", "mtime_epoch": 1782928364.6429813, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 100928}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T02:52:48+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1064.301, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 29, "pushed_frames": 29, "read_frames": 28, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.967, "pushed": 0.967, "read": 0.933}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.452, "count": 28, "max_ms": 4.394, "p95_ms": 3.549}, "gstreamer_push": {"avg_ms": 1.348, "count": 29, "max_ms": 1.437, "p95_ms": 1.412}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.467, "count": 28, "max_ms": 76.422, "p95_ms": 76.387}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.225, "count": 28, "max_ms": 80.184, "p95_ms": 79.971}, "yolo11_640_fire_11class": {"avg_ms": 63.295, "count": 28, "max_ms": 76.198, "p95_ms": 75.937}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.585, "count": 28, "max_ms": 86.361, "p95_ms": 85.596}}, "truncated_count": 0}, "processing_total": {"avg_ms": 82.306, "count": 29, "max_ms": 98.526, "p95_ms": 97.702}, "read_interval": {"avg_ms": 927.262, "count": 28, "max_ms": 23312.03, "p95_ms": 114.764}, "triton_wait": {"avg_ms": 69.657, "count": 28, "max_ms": 86.373, "p95_ms": 85.608}, "visualize": {"avg_ms": 3.676, "count": 28, "max_ms": 4.425, "p95_ms": 3.942}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2403}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4294.523, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55b44371dac0] left block unavailable for requested intra mode [h264 @ 0x55b44371dac0] error while decoding MB 0 22, bytestream 837 [in#0/rtsp @ 0x55b4436d7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b4436d7e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 199, "h264_decode_error_count": 346, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:55:36+09:00", "last_error_summary": "[h264 @ 0x55b44371dac0] left block unavailable for requested intra mode [h264 @ 0x55b44371dac0] error while decoding MB 0 22, bytestream 837 [in#0/rtsp @ 0x55b4436d7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b4436d7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T02:43:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 703.904, "last_started_at": "2026-07-02T02:43:52+09:00", "pid": 33751, "process": {"exists": false, "pid": 33751}, "restart_count": 224, "restart_count_per_hour": 7, "rtsp_timeout_count": 82, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 225}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.116, "sample_count": 2653, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 288452, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:55:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.068, "last_frame_pushed_at": "2026-07-02T02:55:36+09:00", "last_frame_read_age_sec": 0.056, "last_frame_read_at": "2026-07-02T02:55:36+09:00", "mjpeg": {"age_sec": 0.053, "exists": true, "mtime": "2026-07-02T02:55:36.662528+09:00", "mtime_epoch": 1782928536.6625276, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 118958}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T02:55:36+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1002.414, "name": "celery", "pid": 237, "rss_mb": 997.199, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 103, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.401, "pushed": 3.401, "read": 3.435}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.542, "count": 102, "max_ms": 1.974, "p95_ms": 1.573}, "gstreamer_push": {"avg_ms": 1.612, "count": 102, "max_ms": 6.081, "p95_ms": 3.754}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.564, "count": 102, "max_ms": 85.218, "p95_ms": 77.595}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.432, "count": 102, "max_ms": 90.175, "p95_ms": 81.448}, "yolo11_640_fire_11class": {"avg_ms": 70.138, "count": 102, "max_ms": 84.451, "p95_ms": 77.463}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.457, "count": 102, "max_ms": 94.513, "p95_ms": 86.661}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.095, "count": 102, "max_ms": 104.913, "p95_ms": 97.309}, "read_interval": {"avg_ms": 291.635, "count": 103, "max_ms": 19162.906, "p95_ms": 115.418}, "triton_wait": {"avg_ms": 79.539, "count": 102, "max_ms": 94.526, "p95_ms": 86.674}, "visualize": {"avg_ms": 3.712, "count": 102, "max_ms": 4.264, "p95_ms": 3.95}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.989, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2403}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4294.289, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T02:55:36+09:00", "last_error_summary": "[h264 @ 0x55b44371dac0] left block unavailable for requested intra mode [h264 @ 0x55b44371dac0] error while decoding MB 0 22, bytestream 837 [in#0/rtsp @ 0x55b4436d7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b4436d7e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 199, "h264_decode_error_count": 346, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:55:36+09:00", "last_error_summary": "[h264 @ 0x55b44371dac0] left block unavailable for requested intra mode [h264 @ 0x55b44371dac0] error while decoding MB 0 22, bytestream 837 [in#0/rtsp @ 0x55b4436d7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b4436d7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T02:55:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 703.924, "last_started_at": "2026-07-02T02:43:52+09:00", "pid": 33751, "process": {"exists": false, "pid": 33751}, "restart_count": 224, "restart_count_per_hour": 7, "rtsp_timeout_count": 82, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 225}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.116, "sample_count": 2653, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 288453, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:55:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-02T02:55:36+09:00", "last_frame_read_age_sec": 0.077, "last_frame_read_at": "2026-07-02T02:55:36+09:00", "mjpeg": {"age_sec": 0.074, "exists": true, "mtime": "2026-07-02T02:55:36.662528+09:00", "mtime_epoch": 1782928536.6625276, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 118958}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T02:55:36+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1002.414, "name": "celery", "pid": 237, "rss_mb": 997.199, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.443, "pushed": 3.443, "read": 3.409}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.542, "count": 103, "max_ms": 1.974, "p95_ms": 1.573}, "gstreamer_push": {"avg_ms": 1.61, "count": 103, "max_ms": 6.081, "p95_ms": 3.754}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.374, "count": 103, "max_ms": 85.218, "p95_ms": 77.595}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.179, "count": 103, "max_ms": 90.175, "p95_ms": 81.448}, "yolo11_640_fire_11class": {"avg_ms": 69.912, "count": 103, "max_ms": 84.451, "p95_ms": 77.463}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.175, "count": 103, "max_ms": 94.513, "p95_ms": 86.661}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.868, "count": 103, "max_ms": 104.913, "p95_ms": 97.309}, "read_interval": {"avg_ms": 293.457, "count": 102, "max_ms": 19162.906, "p95_ms": 115.418}, "triton_wait": {"avg_ms": 79.256, "count": 103, "max_ms": 94.526, "p95_ms": 86.674}, "visualize": {"avg_ms": 3.746, "count": 103, "max_ms": 7.217, "p95_ms": 4.027}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.919, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2403}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4294.242, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 199, "h264_decode_error_count": 346, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:55:36+09:00", "last_error_summary": "[h264 @ 0x55b44371dac0] left block unavailable for requested intra mode [h264 @ 0x55b44371dac0] error while decoding MB 0 22, bytestream 837 [in#0/rtsp @ 0x55b4436d7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b4436d7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T02:55:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 703.932, "last_started_at": "2026-07-02T02:43:52+09:00", "pid": 33751, "process": {"exists": false, "pid": 33751}, "restart_count": 224, "restart_count_per_hour": 7, "rtsp_timeout_count": 82, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 225}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.116, "sample_count": 2653, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 288453, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:55:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-02T02:55:36+09:00", "last_frame_read_age_sec": 0.085, "last_frame_read_at": "2026-07-02T02:55:36+09:00", "mjpeg": {"age_sec": 0.082, "exists": true, "mtime": "2026-07-02T02:55:36.662528+09:00", "mtime_epoch": 1782928536.6625276, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 118958}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T02:55:36+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1002.414, "name": "celery", "pid": 237, "rss_mb": 997.199, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.442, "pushed": 3.442, "read": 3.408}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.542, "count": 103, "max_ms": 1.974, "p95_ms": 1.573}, "gstreamer_push": {"avg_ms": 1.61, "count": 103, "max_ms": 6.081, "p95_ms": 3.754}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.374, "count": 103, "max_ms": 85.218, "p95_ms": 77.595}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.179, "count": 103, "max_ms": 90.175, "p95_ms": 81.448}, "yolo11_640_fire_11class": {"avg_ms": 69.912, "count": 103, "max_ms": 84.451, "p95_ms": 77.463}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.175, "count": 103, "max_ms": 94.513, "p95_ms": 86.661}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.868, "count": 103, "max_ms": 104.913, "p95_ms": 97.309}, "read_interval": {"avg_ms": 293.457, "count": 102, "max_ms": 19162.906, "p95_ms": 115.418}, "triton_wait": {"avg_ms": 79.256, "count": 103, "max_ms": 94.526, "p95_ms": 86.674}, "visualize": {"avg_ms": 3.746, "count": 103, "max_ms": 7.217, "p95_ms": 4.027}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.928, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2403}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4402.871, "limit_mb": 8796093022207.996}, "detail": {"pid": 34162, "restart_count": 225, "start_count": 226}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 199, "h264_decode_error_count": 346, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:55:36+09:00", "last_error_summary": "[h264 @ 0x55b44371dac0] left block unavailable for requested intra mode [h264 @ 0x55b44371dac0] error while decoding MB 0 22, bytestream 837 [in#0/rtsp @ 0x55b4436d7e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b4436d7e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T02:55:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T02:55:39+09:00", "pid": 34162, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.031, "name": "ffmpeg", "pid": 34162, "rss_mb": 139.031, "state": "S (sleeping)", "threads": 106}, "restart_count": 225, "restart_count_per_hour": 8, "rtsp_timeout_count": 82, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 226}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.116, "sample_count": 2610, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 288453, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:55:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.842, "last_frame_pushed_at": "2026-07-02T02:55:36+09:00", "last_frame_read_age_sec": 4.911, "last_frame_read_at": "2026-07-02T02:55:36+09:00", "mjpeg": {"age_sec": 4.828, "exists": true, "mtime": "2026-07-02T02:55:36.742527+09:00", "mtime_epoch": 1782928536.7425275, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 115977}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T02:55:41+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1002.414, "name": "celery", "pid": 237, "rss_mb": 997.199, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 60, "pushed_frames": 60, "read_frames": 59, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.001, "pushed": 2.001, "read": 1.968}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.546, "count": 60, "max_ms": 1.974, "p95_ms": 1.569}, "gstreamer_push": {"avg_ms": 1.642, "count": 60, "max_ms": 6.081, "p95_ms": 3.754}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.595, "count": 60, "max_ms": 78.26, "p95_ms": 75.303}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.31, "count": 60, "max_ms": 82.073, "p95_ms": 80.108}, "yolo11_640_fire_11class": {"avg_ms": 66.772, "count": 60, "max_ms": 80.318, "p95_ms": 76.288}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.08, "count": 60, "max_ms": 89.086, "p95_ms": 86.578}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.79, "count": 60, "max_ms": 99.434, "p95_ms": 96.898}, "read_interval": {"avg_ms": 426.582, "count": 59, "max_ms": 19162.906, "p95_ms": 115.926}, "triton_wait": {"avg_ms": 76.168, "count": 60, "max_ms": 89.098, "p95_ms": 86.591}, "visualize": {"avg_ms": 3.762, "count": 60, "max_ms": 7.217, "p95_ms": 4.058}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.985, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3406}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4531.641, "limit_mb": 8796093022207.996}, "detail": {"watchdog_timeout_sec": 30.0}, "event_type": "frame_read_watchdog_timeout", "ffmpeg": {"error_count": 49, "h264_decode_error_count": 50, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:52:44+09:00", "last_error_summary": "[h264 @ 0x561b7356c880] error while decoding MB 119 43, bytestream -5 [in#0/rtsp @ 0x561b72bc4e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561b72bc4e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T02:52:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 185.906, "last_started_at": "2026-07-02T02:52:46+09:00", "pid": 34010, "process": {"exists": true, "fd_count": 5, "hwm_mb": 331.504, "name": "ffmpeg", "pid": 34010, "rss_mb": 331.504, "state": "S (sleeping)", "threads": 106}, "restart_count": 49, "restart_count_per_hour": 5, "rtsp_timeout_count": 88, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 50}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.105, "sample_count": 2234, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 408762, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:55:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 30.059, "last_frame_pushed_at": "2026-07-02T02:55:22+09:00", "last_frame_read_age_sec": 30.12, "last_frame_read_at": "2026-07-02T02:55:22+09:00", "mjpeg": {"age_sec": 30.044, "exists": true, "mtime": "2026-07-02T02:55:22.550527+09:00", "mtime_epoch": 1782928522.5505266, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 102450}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T02:55:52+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1062.18, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3406}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4529.734, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 49, "h264_decode_error_count": 50, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:52:44+09:00", "last_error_summary": "[h264 @ 0x561b7356c880] error while decoding MB 119 43, bytestream -5 [in#0/rtsp @ 0x561b72bc4e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561b72bc4e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T02:52:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 185.919, "last_started_at": "2026-07-02T02:52:46+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 49, "restart_count_per_hour": 5, "rtsp_timeout_count": 88, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 50}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.105, "sample_count": 2234, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 408762, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:55:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 30.072, "last_frame_pushed_at": "2026-07-02T02:55:22+09:00", "last_frame_read_age_sec": 30.133, "last_frame_read_at": "2026-07-02T02:55:22+09:00", "mjpeg": {"age_sec": 30.057, "exists": true, "mtime": "2026-07-02T02:55:22.550527+09:00", "mtime_epoch": 1782928522.5505266, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 102450}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T02:55:52+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1062.18, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3406}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4230.707, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x555c6c01e180] error while decoding MB 51 31, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 50, "h264_decode_error_count": 51, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:55:52+09:00", "last_error_summary": "[h264 @ 0x555c6c01e180] error while decoding MB 51 31, bytestream -5", "last_exit_at": "2026-07-02T02:52:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 185.969, "last_started_at": "2026-07-02T02:52:46+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 49, "restart_count_per_hour": 5, "rtsp_timeout_count": 88, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 50}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.105, "sample_count": 2234, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 408762, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:55:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 30.122, "last_frame_pushed_at": "2026-07-02T02:55:22+09:00", "last_frame_read_age_sec": 30.183, "last_frame_read_at": "2026-07-02T02:55:22+09:00", "mjpeg": {"age_sec": 30.107, "exists": true, "mtime": "2026-07-02T02:55:22.550527+09:00", "mtime_epoch": 1782928522.5505266, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 102450}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T02:55:52+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1062.18, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 30, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3406}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4354.484, "limit_mb": 8796093022207.996}, "detail": {"pid": 34269, "restart_count": 50, "start_count": 51}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 50, "h264_decode_error_count": 51, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T02:55:52+09:00", "last_error_summary": "[h264 @ 0x555c6c01e180] error while decoding MB 51 31, bytestream -5", "last_exit_at": "2026-07-02T02:52:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T02:55:54+09:00", "pid": 34269, "process": {"exists": true, "fd_count": 5, "hwm_mb": 157.164, "name": "ffmpeg", "pid": 34269, "rss_mb": 157.023, "state": "R (running)", "threads": 34}, "restart_count": 50, "restart_count_per_hour": 6, "rtsp_timeout_count": 88, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 51}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.105, "sample_count": 2195, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 408762, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T02:55:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s", "last_frame_read_age_sec>30s", "last_frame_pushed_age_sec>30s", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 34.433, "last_frame_pushed_at": "2026-07-02T02:55:22+09:00", "last_frame_read_age_sec": 34.494, "last_frame_read_at": "2026-07-02T02:55:22+09:00", "mjpeg": {"age_sec": 34.418, "exists": true, "mtime": "2026-07-02T02:55:22.550527+09:00", "mtime_epoch": 1782928522.5505266, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 102450}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T02:55:56+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1062.18, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["mjpeg_output_stale_critical", "frame_read_stale_critical", "frame_push_stale_critical", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": ["ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2433}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4299.59, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5595cfee8300] cabac decode of qscale diff failed at 64 44 [h264 @ 0x5595cfee8300] error while decoding MB 64 44, bytestream 3012 [h264 @ 0x5595d0c38980] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5595d0c38980] error while decoding MB 0 25, bytestream 428 [in#0/rtsp @ 0x5595cfe79e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5595cfe79e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 200, "h264_decode_error_count": 349, "h264_decode_error_count_per_hour": 21, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:02:41+09:00", "last_error_summary": "[h264 @ 0x5595cfee8300] cabac decode of qscale diff failed at 64 44 [h264 @ 0x5595cfee8300] error while decoding MB 64 44, bytestream 3012 [h264 @ 0x5595d0c38980] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5595d0c38980] error while decoding MB 0 25, bytestream 428 [in#0/rtsp @ 0x5595cfe79e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5595cfe79e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T02:55:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 421.507, "last_started_at": "2026-07-02T02:55:39+09:00", "pid": 34162, "process": {"exists": false, "pid": 34162}, "restart_count": 225, "restart_count_per_hour": 7, "rtsp_timeout_count": 84, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 226}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.127, "sample_count": 2440, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 292004, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:02:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.072, "last_frame_pushed_at": "2026-07-02T03:02:41+09:00", "last_frame_read_age_sec": 0.06, "last_frame_read_at": "2026-07-02T03:02:41+09:00", "mjpeg": {"age_sec": 0.059, "exists": true, "mtime": "2026-07-02T03:02:41.016742+09:00", "mtime_epoch": 1782928961.016742, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 113423}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T03:02:41+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1005.309, "name": "celery", "pid": 237, "rss_mb": 998.848, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 102, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.405, "pushed": 3.405, "read": 3.405}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.531, "count": 103, "max_ms": 1.877, "p95_ms": 1.569}, "gstreamer_push": {"avg_ms": 1.632, "count": 102, "max_ms": 6.198, "p95_ms": 3.907}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.436, "count": 103, "max_ms": 78.895, "p95_ms": 76.776}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.542, "count": 103, "max_ms": 83.388, "p95_ms": 80.413}, "yolo11_640_fire_11class": {"avg_ms": 70.115, "count": 103, "max_ms": 83.182, "p95_ms": 78.638}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.78, "count": 103, "max_ms": 87.914, "p95_ms": 86.182}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.736, "count": 102, "max_ms": 98.27, "p95_ms": 96.526}, "read_interval": {"avg_ms": 294.043, "count": 102, "max_ms": 19617.673, "p95_ms": 114.426}, "triton_wait": {"avg_ms": 76.082, "count": 103, "max_ms": 87.926, "p95_ms": 86.194}, "visualize": {"avg_ms": 3.675, "count": 103, "max_ms": 4.296, "p95_ms": 3.902}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.958, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2433}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4301.965, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T03:02:41+09:00", "last_error_summary": "[h264 @ 0x5595cfee8300] cabac decode of qscale diff failed at 64 44 [h264 @ 0x5595cfee8300] error while decoding MB 64 44, bytestream 3012 [h264 @ 0x5595d0c38980] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5595d0c38980] error while decoding MB 0 25, bytestream 428 [in#0/rtsp @ 0x5595cfe79e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5595cfe79e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 200, "h264_decode_error_count": 349, "h264_decode_error_count_per_hour": 21, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:02:41+09:00", "last_error_summary": "[h264 @ 0x5595cfee8300] cabac decode of qscale diff failed at 64 44 [h264 @ 0x5595cfee8300] error while decoding MB 64 44, bytestream 3012 [h264 @ 0x5595d0c38980] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5595d0c38980] error while decoding MB 0 25, bytestream 428 [in#0/rtsp @ 0x5595cfe79e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5595cfe79e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:02:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 421.526, "last_started_at": "2026-07-02T02:55:39+09:00", "pid": 34162, "process": {"exists": false, "pid": 34162}, "restart_count": 225, "restart_count_per_hour": 7, "rtsp_timeout_count": 84, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 226}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.127, "sample_count": 2441, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 292004, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:02:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T03:02:41+09:00", "last_frame_read_age_sec": 0.08, "last_frame_read_at": "2026-07-02T03:02:41+09:00", "mjpeg": {"age_sec": 0.079, "exists": true, "mtime": "2026-07-02T03:02:41.016742+09:00", "mtime_epoch": 1782928961.016742, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 113423}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T03:02:41+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1005.309, "name": "celery", "pid": 237, "rss_mb": 1001.426, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.436, "pushed": 3.436, "read": 3.403}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.531, "count": 103, "max_ms": 1.877, "p95_ms": 1.569}, "gstreamer_push": {"avg_ms": 1.655, "count": 103, "max_ms": 6.198, "p95_ms": 3.918}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.436, "count": 103, "max_ms": 78.895, "p95_ms": 76.776}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.542, "count": 103, "max_ms": 83.388, "p95_ms": 80.413}, "yolo11_640_fire_11class": {"avg_ms": 70.115, "count": 103, "max_ms": 83.182, "p95_ms": 78.638}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.78, "count": 103, "max_ms": 87.914, "p95_ms": 86.182}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.558, "count": 103, "max_ms": 98.27, "p95_ms": 96.526}, "read_interval": {"avg_ms": 294.043, "count": 102, "max_ms": 19617.673, "p95_ms": 114.426}, "triton_wait": {"avg_ms": 76.082, "count": 103, "max_ms": 87.926, "p95_ms": 86.194}, "visualize": {"avg_ms": 3.675, "count": 103, "max_ms": 4.296, "p95_ms": 3.902}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.978, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2433}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4301.832, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 200, "h264_decode_error_count": 349, "h264_decode_error_count_per_hour": 21, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:02:41+09:00", "last_error_summary": "[h264 @ 0x5595cfee8300] cabac decode of qscale diff failed at 64 44 [h264 @ 0x5595cfee8300] error while decoding MB 64 44, bytestream 3012 [h264 @ 0x5595d0c38980] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5595d0c38980] error while decoding MB 0 25, bytestream 428 [in#0/rtsp @ 0x5595cfe79e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5595cfe79e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:02:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 421.534, "last_started_at": "2026-07-02T02:55:39+09:00", "pid": 34162, "process": {"exists": false, "pid": 34162}, "restart_count": 225, "restart_count_per_hour": 7, "rtsp_timeout_count": 84, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 226}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.127, "sample_count": 2441, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 292004, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:02:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-02T03:02:41+09:00", "last_frame_read_age_sec": 0.088, "last_frame_read_at": "2026-07-02T03:02:41+09:00", "mjpeg": {"age_sec": 0.087, "exists": true, "mtime": "2026-07-02T03:02:41.016742+09:00", "mtime_epoch": 1782928961.016742, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 113423}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T03:02:41+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1005.309, "name": "celery", "pid": 237, "rss_mb": 1001.426, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.435, "pushed": 3.435, "read": 3.402}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.531, "count": 103, "max_ms": 1.877, "p95_ms": 1.569}, "gstreamer_push": {"avg_ms": 1.655, "count": 103, "max_ms": 6.198, "p95_ms": 3.918}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.436, "count": 103, "max_ms": 78.895, "p95_ms": 76.776}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.542, "count": 103, "max_ms": 83.388, "p95_ms": 80.413}, "yolo11_640_fire_11class": {"avg_ms": 70.115, "count": 103, "max_ms": 83.182, "p95_ms": 78.638}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.78, "count": 103, "max_ms": 87.914, "p95_ms": 86.182}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.558, "count": 103, "max_ms": 98.27, "p95_ms": 96.526}, "read_interval": {"avg_ms": 294.043, "count": 102, "max_ms": 19617.673, "p95_ms": 114.426}, "triton_wait": {"avg_ms": 76.082, "count": 103, "max_ms": 87.926, "p95_ms": 86.194}, "visualize": {"avg_ms": 3.675, "count": 103, "max_ms": 4.296, "p95_ms": 3.902}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.986, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2433}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4409.746, "limit_mb": 8796093022207.996}, "detail": {"pid": 34421, "restart_count": 226, "start_count": 227}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 200, "h264_decode_error_count": 349, "h264_decode_error_count_per_hour": 21, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:02:41+09:00", "last_error_summary": "[h264 @ 0x5595cfee8300] cabac decode of qscale diff failed at 64 44 [h264 @ 0x5595cfee8300] error while decoding MB 64 44, bytestream 3012 [h264 @ 0x5595d0c38980] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5595d0c38980] error while decoding MB 0 25, bytestream 428 [in#0/rtsp @ 0x5595cfe79e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5595cfe79e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:02:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T03:02:43+09:00", "pid": 34421, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.742, "name": "ffmpeg", "pid": 34421, "rss_mb": 138.742, "state": "S (sleeping)", "threads": 106}, "restart_count": 226, "restart_count_per_hour": 8, "rtsp_timeout_count": 84, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 227}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.173, "sample_count": 2385, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 292004, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:02:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.746, "last_frame_pushed_at": "2026-07-02T03:02:41+09:00", "last_frame_read_age_sec": 4.819, "last_frame_read_at": "2026-07-02T03:02:41+09:00", "mjpeg": {"age_sec": 4.731, "exists": true, "mtime": "2026-07-02T03:02:41.104743+09:00", "mtime_epoch": 1782928961.1047428, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 113423}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T03:02:45+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1005.309, "name": "celery", "pid": 237, "rss_mb": 1001.426, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 60, "pushed_frames": 60, "read_frames": 59, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.002, "pushed": 2.002, "read": 1.969}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.53, "count": 60, "max_ms": 1.877, "p95_ms": 1.582}, "gstreamer_push": {"avg_ms": 1.723, "count": 60, "max_ms": 6.198, "p95_ms": 3.963}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.708, "count": 60, "max_ms": 78.895, "p95_ms": 74.067}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.398, "count": 60, "max_ms": 81.556, "p95_ms": 79.575}, "yolo11_640_fire_11class": {"avg_ms": 65.193, "count": 60, "max_ms": 78.364, "p95_ms": 76.156}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.088, "count": 60, "max_ms": 87.914, "p95_ms": 85.587}}, "truncated_count": 0}, "processing_total": {"avg_ms": 81.098, "count": 60, "max_ms": 98.27, "p95_ms": 96.148}, "read_interval": {"avg_ms": 427.777, "count": 59, "max_ms": 19617.673, "p95_ms": 115.655}, "triton_wait": {"avg_ms": 70.538, "count": 60, "max_ms": 87.926, "p95_ms": 85.599}, "visualize": {"avg_ms": 3.689, "count": 60, "max_ms": 4.296, "p95_ms": 3.902}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.964, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3442}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4235.488, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x555f5a132100] error while decoding MB 8 56, bytestream -5 [in#0/rtsp @ 0x555f59fa9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555f59fa9e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 51, "h264_decode_error_count": 52, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:03:55+09:00", "last_error_summary": "[h264 @ 0x555f5a132100] error while decoding MB 8 56, bytestream -5 [in#0/rtsp @ 0x555f59fa9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555f59fa9e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T02:52:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 480.999, "last_started_at": "2026-07-02T02:55:54+09:00", "pid": 34269, "process": {"exists": false, "pid": 34269}, "restart_count": 50, "restart_count_per_hour": 6, "rtsp_timeout_count": 90, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 51}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.864, "sample_count": 2620, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 413040, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:03:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.079, "last_frame_pushed_at": "2026-07-02T03:03:55+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-02T03:03:55+09:00", "mjpeg": {"age_sec": 0.061, "exists": true, "mtime": "2026-07-02T03:03:55.905449+09:00", "mtime_epoch": 1782929035.9054492, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 102188}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:03:55+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1061.605, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.759, "pushed": 0.759, "read": 0.818}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.693, "count": 13, "max_ms": 5.05, "p95_ms": 5.05}, "gstreamer_push": {"avg_ms": 1.377, "count": 13, "max_ms": 1.492, "p95_ms": 1.492}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.229, "count": 13, "max_ms": 73.767, "p95_ms": 73.767}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.007, "count": 13, "max_ms": 75.923, "p95_ms": 75.923}, "yolo11_640_fire_11class": {"avg_ms": 55.345, "count": 13, "max_ms": 76.84, "p95_ms": 76.84}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.981, "count": 13, "max_ms": 85.275, "p95_ms": 85.275}}, "truncated_count": 0}, "processing_total": {"avg_ms": 72.292, "count": 13, "max_ms": 97.469, "p95_ms": 97.469}, "read_interval": {"avg_ms": 2323.881, "count": 14, "max_ms": 16019.378, "p95_ms": 16019.378}, "triton_wait": {"avg_ms": 59.55, "count": 13, "max_ms": 85.286, "p95_ms": 85.286}, "visualize": {"avg_ms": 3.845, "count": 13, "max_ms": 4.793, "p95_ms": 4.793}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.122, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3442}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4234.836, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T03:03:55+09:00", "last_error_summary": "[h264 @ 0x555f5a132100] error while decoding MB 8 56, bytestream -5 [in#0/rtsp @ 0x555f59fa9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555f59fa9e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 51, "h264_decode_error_count": 52, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:03:55+09:00", "last_error_summary": "[h264 @ 0x555f5a132100] error while decoding MB 8 56, bytestream -5 [in#0/rtsp @ 0x555f59fa9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555f59fa9e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:03:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 481.044, "last_started_at": "2026-07-02T02:55:54+09:00", "pid": 34269, "process": {"exists": false, "pid": 34269}, "restart_count": 50, "restart_count_per_hour": 6, "rtsp_timeout_count": 90, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 51}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.864, "sample_count": 2621, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 413041, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:03:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-02T03:03:55+09:00", "last_frame_read_age_sec": 0.11, "last_frame_read_at": "2026-07-02T03:03:55+09:00", "mjpeg": {"age_sec": 0.107, "exists": true, "mtime": "2026-07-02T03:03:55.905449+09:00", "mtime_epoch": 1782929035.9054492, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 102188}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:03:56+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1061.605, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.816, "pushed": 0.816, "read": 0.816}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.668, "count": 14, "max_ms": 5.05, "p95_ms": 5.05}, "gstreamer_push": {"avg_ms": 1.378, "count": 14, "max_ms": 1.492, "p95_ms": 1.492}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.748, "count": 14, "max_ms": 73.767, "p95_ms": 73.767}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.221, "count": 14, "max_ms": 75.923, "p95_ms": 75.923}, "yolo11_640_fire_11class": {"avg_ms": 56.609, "count": 14, "max_ms": 76.84, "p95_ms": 76.84}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.781, "count": 14, "max_ms": 85.275, "p95_ms": 85.275}}, "truncated_count": 0}, "processing_total": {"avg_ms": 73.942, "count": 14, "max_ms": 97.469, "p95_ms": 97.469}, "read_interval": {"avg_ms": 2323.881, "count": 14, "max_ms": 16019.378, "p95_ms": 16019.378}, "triton_wait": {"avg_ms": 61.239, "count": 14, "max_ms": 85.286, "p95_ms": 85.286}, "visualize": {"avg_ms": 3.848, "count": 14, "max_ms": 4.793, "p95_ms": 4.793}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.167, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3442}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4234.77, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 51, "h264_decode_error_count": 52, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:03:55+09:00", "last_error_summary": "[h264 @ 0x555f5a132100] error while decoding MB 8 56, bytestream -5 [in#0/rtsp @ 0x555f59fa9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555f59fa9e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:03:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 481.054, "last_started_at": "2026-07-02T02:55:54+09:00", "pid": 34269, "process": {"exists": false, "pid": 34269}, "restart_count": 50, "restart_count_per_hour": 6, "rtsp_timeout_count": 90, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 51}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.864, "sample_count": 2621, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 413041, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:03:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.022, "last_frame_pushed_at": "2026-07-02T03:03:55+09:00", "last_frame_read_age_sec": 0.12, "last_frame_read_at": "2026-07-02T03:03:55+09:00", "mjpeg": {"age_sec": 0.009, "exists": true, "mtime": "2026-07-02T03:03:56.013450+09:00", "mtime_epoch": 1782929036.0134501, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 102484}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:03:56+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1061.605, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.815, "pushed": 0.815, "read": 0.815}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.668, "count": 14, "max_ms": 5.05, "p95_ms": 5.05}, "gstreamer_push": {"avg_ms": 1.378, "count": 14, "max_ms": 1.492, "p95_ms": 1.492}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.748, "count": 14, "max_ms": 73.767, "p95_ms": 73.767}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.221, "count": 14, "max_ms": 75.923, "p95_ms": 75.923}, "yolo11_640_fire_11class": {"avg_ms": 56.609, "count": 14, "max_ms": 76.84, "p95_ms": 76.84}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.781, "count": 14, "max_ms": 85.275, "p95_ms": 85.275}}, "truncated_count": 0}, "processing_total": {"avg_ms": 73.942, "count": 14, "max_ms": 97.469, "p95_ms": 97.469}, "read_interval": {"avg_ms": 2323.881, "count": 14, "max_ms": 16019.378, "p95_ms": 16019.378}, "triton_wait": {"avg_ms": 61.239, "count": 14, "max_ms": 85.286, "p95_ms": 85.286}, "visualize": {"avg_ms": 3.848, "count": 14, "max_ms": 4.793, "p95_ms": 4.793}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.177, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3442}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4255.605, "limit_mb": 8796093022207.996}, "detail": {"pid": 34528, "restart_count": 51, "start_count": 52}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 51, "h264_decode_error_count": 52, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:03:55+09:00", "last_error_summary": "[h264 @ 0x555f5a132100] error while decoding MB 8 56, bytestream -5 [in#0/rtsp @ 0x555f59fa9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555f59fa9e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:03:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T03:03:58+09:00", "pid": 34528, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.285, "name": "ffmpeg", "pid": 34528, "rss_mb": 58.285, "state": "S (sleeping)", "threads": 1}, "restart_count": 51, "restart_count_per_hour": 7, "rtsp_timeout_count": 90, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 52}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.823, "sample_count": 2579, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 413041, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:03:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.682, "last_frame_pushed_at": "2026-07-02T03:03:55+09:00", "last_frame_read_age_sec": 4.78, "last_frame_read_at": "2026-07-02T03:03:55+09:00", "mjpeg": {"age_sec": 4.668, "exists": true, "mtime": "2026-07-02T03:03:56.013450+09:00", "mtime_epoch": 1782929036.0134501, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 102484}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:04:00+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1061.605, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.641, "pushed": 0.641, "read": 0.641}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.668, "count": 14, "max_ms": 5.05, "p95_ms": 5.05}, "gstreamer_push": {"avg_ms": 1.378, "count": 14, "max_ms": 1.492, "p95_ms": 1.492}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.748, "count": 14, "max_ms": 73.767, "p95_ms": 73.767}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.221, "count": 14, "max_ms": 75.923, "p95_ms": 75.923}, "yolo11_640_fire_11class": {"avg_ms": 56.609, "count": 14, "max_ms": 76.84, "p95_ms": 76.84}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.781, "count": 14, "max_ms": 85.275, "p95_ms": 85.275}}, "truncated_count": 0}, "processing_total": {"avg_ms": 73.942, "count": 14, "max_ms": 97.469, "p95_ms": 97.469}, "read_interval": {"avg_ms": 2323.881, "count": 14, "max_ms": 16019.378, "p95_ms": 16019.378}, "triton_wait": {"avg_ms": 61.239, "count": 14, "max_ms": 85.286, "p95_ms": 85.286}, "visualize": {"avg_ms": 3.848, "count": 14, "max_ms": 4.793, "p95_ms": 4.793}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.837, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3459}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4245.977, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5612001c7840] error while decoding MB 85 35, bytestream -5 [in#0/rtsp @ 0x5611ff4e3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5611ff4e3e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 52, "h264_decode_error_count": 53, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:08:07+09:00", "last_error_summary": "[h264 @ 0x5612001c7840] error while decoding MB 85 35, bytestream -5 [in#0/rtsp @ 0x5611ff4e3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5611ff4e3e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:03:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 249.155, "last_started_at": "2026-07-02T03:03:58+09:00", "pid": 34528, "process": {"exists": false, "pid": 34528}, "restart_count": 51, "restart_count_per_hour": 7, "rtsp_timeout_count": 92, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 52}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.647, "sample_count": 2194, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 415081, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:08:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.076, "last_frame_pushed_at": "2026-07-02T03:08:07+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-02T03:08:07+09:00", "mjpeg": {"age_sec": 0.06, "exists": true, "mtime": "2026-07-02T03:08:07.776248+09:00", "mtime_epoch": 1782929287.7762485, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 104056}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:08:07+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1065.273, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 67, "pushed_frames": 67, "read_frames": 67, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.237, "pushed": 2.237, "read": 2.237}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.598, "count": 68, "max_ms": 5.703, "p95_ms": 5.387}, "gstreamer_push": {"avg_ms": 1.519, "count": 67, "max_ms": 3.865, "p95_ms": 3.728}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.664, "count": 68, "max_ms": 82.063, "p95_ms": 78.133}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.608, "count": 68, "max_ms": 83.179, "p95_ms": 80.025}, "yolo11_640_fire_11class": {"avg_ms": 70.307, "count": 68, "max_ms": 82.675, "p95_ms": 79.357}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.428, "count": 68, "max_ms": 91.321, "p95_ms": 87.391}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.378, "count": 67, "max_ms": 103.721, "p95_ms": 100.118}, "read_interval": {"avg_ms": 447.518, "count": 67, "max_ms": 22918.584, "p95_ms": 121.182}, "triton_wait": {"avg_ms": 77.201, "count": 68, "max_ms": 91.335, "p95_ms": 87.404}, "visualize": {"avg_ms": 3.9, "count": 68, "max_ms": 6.551, "p95_ms": 6.176}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.951, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3459}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4245.766, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T03:08:07+09:00", "last_error_summary": "[h264 @ 0x5612001c7840] error while decoding MB 85 35, bytestream -5 [in#0/rtsp @ 0x5611ff4e3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5611ff4e3e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 52, "h264_decode_error_count": 53, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:08:07+09:00", "last_error_summary": "[h264 @ 0x5612001c7840] error while decoding MB 85 35, bytestream -5 [in#0/rtsp @ 0x5611ff4e3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5611ff4e3e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:08:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 249.169, "last_started_at": "2026-07-02T03:03:58+09:00", "pid": 34528, "process": {"exists": false, "pid": 34528}, "restart_count": 51, "restart_count_per_hour": 7, "rtsp_timeout_count": 92, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 52}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.647, "sample_count": 2195, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 415081, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:08:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T03:08:07+09:00", "last_frame_read_age_sec": 0.077, "last_frame_read_at": "2026-07-02T03:08:07+09:00", "mjpeg": {"age_sec": 0.073, "exists": true, "mtime": "2026-07-02T03:08:07.776248+09:00", "mtime_epoch": 1782929287.7762485, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 104056}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:08:07+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1065.273, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 68, "pushed_frames": 68, "read_frames": 67, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.269, "pushed": 2.269, "read": 2.236}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.598, "count": 68, "max_ms": 5.703, "p95_ms": 5.387}, "gstreamer_push": {"avg_ms": 1.518, "count": 68, "max_ms": 3.865, "p95_ms": 3.728}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.664, "count": 68, "max_ms": 82.063, "p95_ms": 78.133}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.608, "count": 68, "max_ms": 83.179, "p95_ms": 80.025}, "yolo11_640_fire_11class": {"avg_ms": 70.307, "count": 68, "max_ms": 82.675, "p95_ms": 79.357}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.428, "count": 68, "max_ms": 91.321, "p95_ms": 87.391}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.037, "count": 68, "max_ms": 103.721, "p95_ms": 100.118}, "read_interval": {"avg_ms": 447.518, "count": 67, "max_ms": 22918.584, "p95_ms": 121.182}, "triton_wait": {"avg_ms": 77.201, "count": 68, "max_ms": 91.335, "p95_ms": 87.404}, "visualize": {"avg_ms": 3.9, "count": 68, "max_ms": 6.551, "p95_ms": 6.176}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.965, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3459}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4245.598, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 52, "h264_decode_error_count": 53, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:08:07+09:00", "last_error_summary": "[h264 @ 0x5612001c7840] error while decoding MB 85 35, bytestream -5 [in#0/rtsp @ 0x5611ff4e3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5611ff4e3e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:08:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 249.177, "last_started_at": "2026-07-02T03:03:58+09:00", "pid": 34528, "process": {"exists": false, "pid": 34528}, "restart_count": 51, "restart_count_per_hour": 7, "rtsp_timeout_count": 92, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 52}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.647, "sample_count": 2195, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 415081, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:08:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-02T03:08:07+09:00", "last_frame_read_age_sec": 0.085, "last_frame_read_at": "2026-07-02T03:08:07+09:00", "mjpeg": {"age_sec": 0.081, "exists": true, "mtime": "2026-07-02T03:08:07.776248+09:00", "mtime_epoch": 1782929287.7762485, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 104056}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:08:07+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1065.273, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 68, "pushed_frames": 68, "read_frames": 67, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.269, "pushed": 2.269, "read": 2.235}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.598, "count": 68, "max_ms": 5.703, "p95_ms": 5.387}, "gstreamer_push": {"avg_ms": 1.518, "count": 68, "max_ms": 3.865, "p95_ms": 3.728}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.664, "count": 68, "max_ms": 82.063, "p95_ms": 78.133}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.608, "count": 68, "max_ms": 83.179, "p95_ms": 80.025}, "yolo11_640_fire_11class": {"avg_ms": 70.307, "count": 68, "max_ms": 82.675, "p95_ms": 79.357}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.428, "count": 68, "max_ms": 91.321, "p95_ms": 87.391}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.037, "count": 68, "max_ms": 103.721, "p95_ms": 100.118}, "read_interval": {"avg_ms": 447.518, "count": 67, "max_ms": 22918.584, "p95_ms": 121.182}, "triton_wait": {"avg_ms": 77.201, "count": 68, "max_ms": 91.335, "p95_ms": 87.404}, "visualize": {"avg_ms": 3.9, "count": 68, "max_ms": 6.551, "p95_ms": 6.176}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.973, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3459}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4266.035, "limit_mb": 8796093022207.996}, "detail": {"pid": 34680, "restart_count": 52, "start_count": 53}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 52, "h264_decode_error_count": 53, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:08:07+09:00", "last_error_summary": "[h264 @ 0x5612001c7840] error while decoding MB 85 35, bytestream -5 [in#0/rtsp @ 0x5611ff4e3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5611ff4e3e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:08:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T03:08:10+09:00", "pid": 34680, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.535, "name": "ffmpeg", "pid": 34680, "rss_mb": 57.535, "state": "S (sleeping)", "threads": 1}, "restart_count": 52, "restart_count_per_hour": 8, "rtsp_timeout_count": 92, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 53}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.651, "sample_count": 2153, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 415081, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:08:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.632, "last_frame_pushed_at": "2026-07-02T03:08:07+09:00", "last_frame_read_age_sec": 4.702, "last_frame_read_at": "2026-07-02T03:08:07+09:00", "mjpeg": {"age_sec": 4.618, "exists": true, "mtime": "2026-07-02T03:08:07.856249+09:00", "mtime_epoch": 1782929287.8562493, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 95347}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:08:12+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1065.273, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 27, "pushed_frames": 27, "read_frames": 26, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.901, "pushed": 0.901, "read": 0.867}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.757, "count": 27, "max_ms": 5.703, "p95_ms": 5.544}, "gstreamer_push": {"avg_ms": 1.612, "count": 27, "max_ms": 3.865, "p95_ms": 3.728}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.296, "count": 27, "max_ms": 75.066, "p95_ms": 71.91}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.711, "count": 27, "max_ms": 80.467, "p95_ms": 76.242}, "yolo11_640_fire_11class": {"avg_ms": 62.545, "count": 27, "max_ms": 78.217, "p95_ms": 76.88}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.114, "count": 27, "max_ms": 87.215, "p95_ms": 84.421}}, "truncated_count": 0}, "processing_total": {"avg_ms": 81.735, "count": 27, "max_ms": 101.853, "p95_ms": 100.049}, "read_interval": {"avg_ms": 975.941, "count": 26, "max_ms": 22918.584, "p95_ms": 121.182}, "triton_wait": {"avg_ms": 68.243, "count": 27, "max_ms": 87.226, "p95_ms": 84.433}, "visualize": {"avg_ms": 4.109, "count": 27, "max_ms": 6.551, "p95_ms": 6.223}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.983, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3477}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4251.273, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x565522dde200] missing picture in access unit with size 18 [h264 @ 0x5655239ec9c0] no frame! [in#0/rtsp @ 0x565522dd9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x565522dd9e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x565522dde840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 53, "h264_decode_error_count": 54, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:12:47+09:00", "last_error_summary": "[NULL @ 0x565522dde200] missing picture in access unit with size 18 [h264 @ 0x5655239ec9c0] no frame! [in#0/rtsp @ 0x565522dd9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x565522dd9e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x565522dde840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T03:08:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 277.009, "last_started_at": "2026-07-02T03:08:10+09:00", "pid": 34680, "process": {"exists": false, "pid": 34680}, "restart_count": 52, "restart_count_per_hour": 8, "rtsp_timeout_count": 94, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 53}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.813, "sample_count": 2229, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 417297, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:12:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.075, "last_frame_pushed_at": "2026-07-02T03:12:47+09:00", "last_frame_read_age_sec": 0.063, "last_frame_read_at": "2026-07-02T03:12:47+09:00", "mjpeg": {"age_sec": 0.058, "exists": true, "mtime": "2026-07-02T03:12:47.423875+09:00", "mtime_epoch": 1782929567.4238749, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 100354}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:12:47+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1069.047, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.47, "pushed": 0.434, "read": 0.47}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.641, "count": 13, "max_ms": 5.148, "p95_ms": 5.148}, "gstreamer_push": {"avg_ms": 1.542, "count": 12, "max_ms": 3.607, "p95_ms": 3.607}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.144, "count": 13, "max_ms": 55.548, "p95_ms": 55.548}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.546, "count": 13, "max_ms": 70.629, "p95_ms": 70.629}, "yolo11_640_fire_11class": {"avg_ms": 53.071, "count": 13, "max_ms": 70.617, "p95_ms": 70.617}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.443, "count": 13, "max_ms": 82.834, "p95_ms": 82.834}}, "truncated_count": 0}, "processing_total": {"avg_ms": 69.1, "count": 13, "max_ms": 95.03, "p95_ms": 95.03}, "read_interval": {"avg_ms": 2600.565, "count": 13, "max_ms": 26679.417, "p95_ms": 26679.417}, "triton_wait": {"avg_ms": 56.576, "count": 13, "max_ms": 82.846, "p95_ms": 82.846}, "visualize": {"avg_ms": 3.721, "count": 13, "max_ms": 4.162, "p95_ms": 4.162}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.647, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3477}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4251.246, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T03:12:47+09:00", "last_error_summary": "[NULL @ 0x565522dde200] missing picture in access unit with size 18 [h264 @ 0x5655239ec9c0] no frame! [in#0/rtsp @ 0x565522dd9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x565522dd9e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x565522dde840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 53, "h264_decode_error_count": 54, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:12:47+09:00", "last_error_summary": "[NULL @ 0x565522dde200] missing picture in access unit with size 18 [h264 @ 0x5655239ec9c0] no frame! [in#0/rtsp @ 0x565522dd9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x565522dd9e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x565522dde840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T03:12:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 277.024, "last_started_at": "2026-07-02T03:08:10+09:00", "pid": 34680, "process": {"exists": false, "pid": 34680}, "restart_count": 52, "restart_count_per_hour": 8, "rtsp_timeout_count": 94, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 53}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.813, "sample_count": 2229, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 417297, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:12:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-02T03:12:47+09:00", "last_frame_read_age_sec": 0.078, "last_frame_read_at": "2026-07-02T03:12:47+09:00", "mjpeg": {"age_sec": 0.073, "exists": true, "mtime": "2026-07-02T03:12:47.423875+09:00", "mtime_epoch": 1782929567.4238749, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 100354}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:12:47+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1069.047, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.47, "pushed": 0.47, "read": 0.47}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.641, "count": 13, "max_ms": 5.148, "p95_ms": 5.148}, "gstreamer_push": {"avg_ms": 1.74, "count": 13, "max_ms": 4.113, "p95_ms": 4.113}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.144, "count": 13, "max_ms": 55.548, "p95_ms": 55.548}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.546, "count": 13, "max_ms": 70.629, "p95_ms": 70.629}, "yolo11_640_fire_11class": {"avg_ms": 53.071, "count": 13, "max_ms": 70.617, "p95_ms": 70.617}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.443, "count": 13, "max_ms": 82.834, "p95_ms": 82.834}}, "truncated_count": 0}, "processing_total": {"avg_ms": 69.1, "count": 13, "max_ms": 95.03, "p95_ms": 95.03}, "read_interval": {"avg_ms": 2600.565, "count": 13, "max_ms": 26679.417, "p95_ms": 26679.417}, "triton_wait": {"avg_ms": 56.576, "count": 13, "max_ms": 82.846, "p95_ms": 82.846}, "visualize": {"avg_ms": 3.721, "count": 13, "max_ms": 4.162, "p95_ms": 4.162}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.661, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3477}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4251.062, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 53, "h264_decode_error_count": 54, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:12:47+09:00", "last_error_summary": "[NULL @ 0x565522dde200] missing picture in access unit with size 18 [h264 @ 0x5655239ec9c0] no frame! [in#0/rtsp @ 0x565522dd9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x565522dd9e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x565522dde840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T03:12:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 277.031, "last_started_at": "2026-07-02T03:08:10+09:00", "pid": 34680, "process": {"exists": false, "pid": 34680}, "restart_count": 52, "restart_count_per_hour": 8, "rtsp_timeout_count": 94, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 53}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.813, "sample_count": 2229, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 417297, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:12:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.019, "last_frame_pushed_at": "2026-07-02T03:12:47+09:00", "last_frame_read_age_sec": 0.086, "last_frame_read_at": "2026-07-02T03:12:47+09:00", "mjpeg": {"age_sec": 0.005, "exists": true, "mtime": "2026-07-02T03:12:47.499876+09:00", "mtime_epoch": 1782929567.499876, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101486}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:12:47+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1069.047, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.47, "pushed": 0.47, "read": 0.47}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.641, "count": 13, "max_ms": 5.148, "p95_ms": 5.148}, "gstreamer_push": {"avg_ms": 1.74, "count": 13, "max_ms": 4.113, "p95_ms": 4.113}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.144, "count": 13, "max_ms": 55.548, "p95_ms": 55.548}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.546, "count": 13, "max_ms": 70.629, "p95_ms": 70.629}, "yolo11_640_fire_11class": {"avg_ms": 53.071, "count": 13, "max_ms": 70.617, "p95_ms": 70.617}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.443, "count": 13, "max_ms": 82.834, "p95_ms": 82.834}}, "truncated_count": 0}, "processing_total": {"avg_ms": 69.1, "count": 13, "max_ms": 95.03, "p95_ms": 95.03}, "read_interval": {"avg_ms": 2600.565, "count": 13, "max_ms": 26679.417, "p95_ms": 26679.417}, "triton_wait": {"avg_ms": 56.576, "count": 13, "max_ms": 82.846, "p95_ms": 82.846}, "visualize": {"avg_ms": 3.721, "count": 13, "max_ms": 4.162, "p95_ms": 4.162}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.669, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3477}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4425.199, "limit_mb": 8796093022207.996}, "detail": {"pid": 34832, "restart_count": 53, "start_count": 54}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 53, "h264_decode_error_count": 54, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:12:47+09:00", "last_error_summary": "[NULL @ 0x565522dde200] missing picture in access unit with size 18 [h264 @ 0x5655239ec9c0] no frame! [in#0/rtsp @ 0x565522dd9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x565522dd9e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x565522dde840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T03:12:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T03:12:50+09:00", "pid": 34832, "process": {"exists": true, "fd_count": 5, "hwm_mb": 201.559, "name": "ffmpeg", "pid": 34832, "rss_mb": 201.559, "state": "S (sleeping)", "threads": 106}, "restart_count": 53, "restart_count_per_hour": 9, "rtsp_timeout_count": 94, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 54}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.813, "sample_count": 2229, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 417297, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:12:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.631, "last_frame_pushed_at": "2026-07-02T03:12:47+09:00", "last_frame_read_age_sec": 4.698, "last_frame_read_at": "2026-07-02T03:12:47+09:00", "mjpeg": {"age_sec": 4.617, "exists": true, "mtime": "2026-07-02T03:12:47.499876+09:00", "mtime_epoch": 1782929567.499876, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 101486}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:12:52+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1069.047, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 12, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.142, "pushed": 2.142, "read": 2.142}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.664, "count": 12, "max_ms": 5.148, "p95_ms": 5.148}, "gstreamer_push": {"avg_ms": 1.778, "count": 12, "max_ms": 4.113, "p95_ms": 4.113}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.277, "count": 12, "max_ms": 54.037, "p95_ms": 54.037}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.289, "count": 12, "max_ms": 59.83, "p95_ms": 59.83}, "yolo11_640_fire_11class": {"avg_ms": 51.609, "count": 12, "max_ms": 54.569, "p95_ms": 54.569}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.077, "count": 12, "max_ms": 55.119, "p95_ms": 55.119}}, "truncated_count": 0}, "processing_total": {"avg_ms": 66.94, "count": 12, "max_ms": 71.992, "p95_ms": 71.992}, "read_interval": {"avg_ms": 2298.574, "count": 12, "max_ms": 26679.417, "p95_ms": 26679.417}, "triton_wait": {"avg_ms": 54.387, "count": 12, "max_ms": 59.842, "p95_ms": 59.842}, "visualize": {"avg_ms": 3.734, "count": 12, "max_ms": 4.162, "p95_ms": 4.162}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 5.602, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3489}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4279.84, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55bfb12e0480] error while decoding MB 81 8, bytestream -5 [in#0/rtsp @ 0x55bfb0797e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55bfb0797e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 54, "h264_decode_error_count": 55, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:15:54+09:00", "last_error_summary": "[h264 @ 0x55bfb12e0480] error while decoding MB 81 8, bytestream -5 [in#0/rtsp @ 0x55bfb0797e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55bfb0797e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:12:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 184.871, "last_started_at": "2026-07-02T03:12:50+09:00", "pid": 34832, "process": {"exists": false, "pid": 34832}, "restart_count": 53, "restart_count_per_hour": 9, "rtsp_timeout_count": 96, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 54}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.458, "sample_count": 2142, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 418706, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:15:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.076, "last_frame_pushed_at": "2026-07-02T03:15:54+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-02T03:15:54+09:00", "mjpeg": {"age_sec": 0.06, "exists": true, "mtime": "2026-07-02T03:15:54.926497+09:00", "mtime_epoch": 1782929754.9264967, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 105193}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:15:54+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1064.559, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 15, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.501, "pushed": 0.468, "read": 0.468}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.371, "count": 15, "max_ms": 3.49, "p95_ms": 3.49}, "gstreamer_push": {"avg_ms": 1.663, "count": 14, "max_ms": 5.924, "p95_ms": 5.924}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.233, "count": 15, "max_ms": 71.683, "p95_ms": 71.683}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.363, "count": 15, "max_ms": 77.314, "p95_ms": 77.314}, "yolo11_640_fire_11class": {"avg_ms": 52.183, "count": 15, "max_ms": 77.327, "p95_ms": 77.327}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.19, "count": 15, "max_ms": 83.41, "p95_ms": 83.41}}, "truncated_count": 0}, "processing_total": {"avg_ms": 67.038, "count": 15, "max_ms": 95.649, "p95_ms": 95.649}, "read_interval": {"avg_ms": 2139.052, "count": 14, "max_ms": 16052.758, "p95_ms": 16052.758}, "triton_wait": {"avg_ms": 54.874, "count": 15, "max_ms": 83.423, "p95_ms": 83.423}, "visualize": {"avg_ms": 3.717, "count": 15, "max_ms": 4.055, "p95_ms": 4.055}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.917, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3489}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4279.809, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T03:15:54+09:00", "last_error_summary": "[h264 @ 0x55bfb12e0480] error while decoding MB 81 8, bytestream -5 [in#0/rtsp @ 0x55bfb0797e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55bfb0797e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 54, "h264_decode_error_count": 55, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:15:54+09:00", "last_error_summary": "[h264 @ 0x55bfb12e0480] error while decoding MB 81 8, bytestream -5 [in#0/rtsp @ 0x55bfb0797e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55bfb0797e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:15:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 184.886, "last_started_at": "2026-07-02T03:12:50+09:00", "pid": 34832, "process": {"exists": false, "pid": 34832}, "restart_count": 53, "restart_count_per_hour": 9, "rtsp_timeout_count": 96, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 54}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.458, "sample_count": 2142, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 418706, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:15:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-02T03:15:54+09:00", "last_frame_read_age_sec": 0.08, "last_frame_read_at": "2026-07-02T03:15:54+09:00", "mjpeg": {"age_sec": 0.076, "exists": true, "mtime": "2026-07-02T03:15:54.926497+09:00", "mtime_epoch": 1782929754.9264967, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 105193}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:15:55+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1064.559, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 15, "pushed_frames": 15, "read_frames": 14, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.501, "pushed": 0.501, "read": 0.468}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.371, "count": 15, "max_ms": 3.49, "p95_ms": 3.49}, "gstreamer_push": {"avg_ms": 1.812, "count": 15, "max_ms": 5.924, "p95_ms": 5.924}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.233, "count": 15, "max_ms": 71.683, "p95_ms": 71.683}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.363, "count": 15, "max_ms": 77.314, "p95_ms": 77.314}, "yolo11_640_fire_11class": {"avg_ms": 52.183, "count": 15, "max_ms": 77.327, "p95_ms": 77.327}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.19, "count": 15, "max_ms": 83.41, "p95_ms": 83.41}}, "truncated_count": 0}, "processing_total": {"avg_ms": 67.038, "count": 15, "max_ms": 95.649, "p95_ms": 95.649}, "read_interval": {"avg_ms": 2139.052, "count": 14, "max_ms": 16052.758, "p95_ms": 16052.758}, "triton_wait": {"avg_ms": 54.874, "count": 15, "max_ms": 83.423, "p95_ms": 83.423}, "visualize": {"avg_ms": 3.717, "count": 15, "max_ms": 4.055, "p95_ms": 4.055}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.931, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3489}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4279.73, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 54, "h264_decode_error_count": 55, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:15:54+09:00", "last_error_summary": "[h264 @ 0x55bfb12e0480] error while decoding MB 81 8, bytestream -5 [in#0/rtsp @ 0x55bfb0797e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55bfb0797e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:15:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 184.893, "last_started_at": "2026-07-02T03:12:50+09:00", "pid": 34832, "process": {"exists": false, "pid": 34832}, "restart_count": 53, "restart_count_per_hour": 9, "rtsp_timeout_count": 96, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 54}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.458, "sample_count": 2142, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 418706, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:15:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.019, "last_frame_pushed_at": "2026-07-02T03:15:54+09:00", "last_frame_read_age_sec": 0.087, "last_frame_read_at": "2026-07-02T03:15:54+09:00", "mjpeg": {"age_sec": 0.007, "exists": true, "mtime": "2026-07-02T03:15:55.002498+09:00", "mtime_epoch": 1782929755.002498, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 105384}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:15:55+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1064.559, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 15, "pushed_frames": 15, "read_frames": 14, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.501, "pushed": 0.501, "read": 0.468}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.371, "count": 15, "max_ms": 3.49, "p95_ms": 3.49}, "gstreamer_push": {"avg_ms": 1.812, "count": 15, "max_ms": 5.924, "p95_ms": 5.924}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.233, "count": 15, "max_ms": 71.683, "p95_ms": 71.683}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.363, "count": 15, "max_ms": 77.314, "p95_ms": 77.314}, "yolo11_640_fire_11class": {"avg_ms": 52.183, "count": 15, "max_ms": 77.327, "p95_ms": 77.327}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.19, "count": 15, "max_ms": 83.41, "p95_ms": 83.41}}, "truncated_count": 0}, "processing_total": {"avg_ms": 67.038, "count": 15, "max_ms": 95.649, "p95_ms": 95.649}, "read_interval": {"avg_ms": 2139.052, "count": 14, "max_ms": 16052.758, "p95_ms": 16052.758}, "triton_wait": {"avg_ms": 54.874, "count": 15, "max_ms": 83.423, "p95_ms": 83.423}, "visualize": {"avg_ms": 3.717, "count": 15, "max_ms": 4.055, "p95_ms": 4.055}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.938, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3489}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4459.301, "limit_mb": 8796093022207.996}, "detail": {"pid": 34984, "restart_count": 54, "start_count": 55}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 54, "h264_decode_error_count": 55, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:15:54+09:00", "last_error_summary": "[h264 @ 0x55bfb12e0480] error while decoding MB 81 8, bytestream -5 [in#0/rtsp @ 0x55bfb0797e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55bfb0797e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:15:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T03:15:57+09:00", "pid": 34984, "process": {"exists": true, "fd_count": 5, "hwm_mb": 209.188, "name": "ffmpeg", "pid": 34984, "rss_mb": 209.188, "state": "S (sleeping)", "threads": 106}, "restart_count": 54, "restart_count_per_hour": 10, "rtsp_timeout_count": 96, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 55}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.471, "sample_count": 2102, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 418706, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:15:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.341, "last_frame_pushed_at": "2026-07-02T03:15:54+09:00", "last_frame_read_age_sec": 4.409, "last_frame_read_at": "2026-07-02T03:15:54+09:00", "mjpeg": {"age_sec": 4.33, "exists": true, "mtime": "2026-07-02T03:15:55.002498+09:00", "mtime_epoch": 1782929755.002498, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 105384}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:15:59+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1064.559, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.611, "pushed": 0.611, "read": 0.611}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.366, "count": 13, "max_ms": 3.49, "p95_ms": 3.49}, "gstreamer_push": {"avg_ms": 1.885, "count": 13, "max_ms": 5.924, "p95_ms": 5.924}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.09, "count": 13, "max_ms": 71.683, "p95_ms": 71.683}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.753, "count": 13, "max_ms": 73.907, "p95_ms": 73.907}, "yolo11_640_fire_11class": {"avg_ms": 48.467, "count": 13, "max_ms": 72.443, "p95_ms": 72.443}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.703, "count": 13, "max_ms": 79.657, "p95_ms": 79.657}}, "truncated_count": 0}, "processing_total": {"avg_ms": 62.657, "count": 13, "max_ms": 92.217, "p95_ms": 92.217}, "read_interval": {"avg_ms": 2295.031, "count": 13, "max_ms": 16052.758, "p95_ms": 16052.758}, "triton_wait": {"avg_ms": 50.491, "count": 13, "max_ms": 79.669, "p95_ms": 79.669}, "visualize": {"avg_ms": 3.733, "count": 13, "max_ms": 4.055, "p95_ms": 4.055}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.278, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2497}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4305.559, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 7, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55e397acb7c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e397acb7c0] error while decoding MB 0 21, bytestream 1372 [h264 @ 0x55e396e29180] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e396e29180] error while decoding MB 0 30, bytestream 7259 [h264 @ 0x55e396e15140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e396e15140] error while decoding MB 0 4, bytestream 6801 [h264 @ 0x55e39792dac0] left block unavailable for reques", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 201, "h264_decode_error_count": 356, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:16:14+09:00", "last_error_summary": "[h264 @ 0x55e397acb7c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e397acb7c0] error while decoding MB 0 21, bytestream 1372 [h264 @ 0x55e396e29180] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e396e29180] error while decoding MB 0 30, bytestream 7259 [h264 @ 0x55e396e15140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e396e15140] error while decoding MB 0 4, bytestream 6801 [h264 @ 0x55e39792dac0] left block unavailable for reques", "last_exit_at": "2026-07-02T03:02:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 810.478, "last_started_at": "2026-07-02T03:02:43+09:00", "pid": 34421, "process": {"exists": false, "pid": 34421}, "restart_count": 226, "restart_count_per_hour": 5, "rtsp_timeout_count": 86, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 227}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 96.993, "sample_count": 2813, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 299644, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:16:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.07, "last_frame_pushed_at": "2026-07-02T03:16:14+09:00", "last_frame_read_age_sec": 0.056, "last_frame_read_at": "2026-07-02T03:16:14+09:00", "mjpeg": {"age_sec": 0.053, "exists": true, "mtime": "2026-07-02T03:16:14.258773+09:00", "mtime_epoch": 1782929774.2587733, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 112838}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T03:16:14+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1007.098, "name": "celery", "pid": 237, "rss_mb": 1005.035, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 130, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.319, "pushed": 4.319, "read": 4.352}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.529, "count": 129, "max_ms": 1.873, "p95_ms": 1.584}, "gstreamer_push": {"avg_ms": 1.521, "count": 129, "max_ms": 6.232, "p95_ms": 3.652}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.573, "count": 129, "max_ms": 74.59, "p95_ms": 52.215}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.193, "count": 129, "max_ms": 83.827, "p95_ms": 74.863}, "yolo11_640_fire_11class": {"avg_ms": 48.113, "count": 129, "max_ms": 75.581, "p95_ms": 72.876}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.709, "count": 129, "max_ms": 85.078, "p95_ms": 81.944}}, "truncated_count": 0}, "processing_total": {"avg_ms": 60.316, "count": 129, "max_ms": 95.458, "p95_ms": 92.236}, "read_interval": {"avg_ms": 237.793, "count": 130, "max_ms": 10124.532, "p95_ms": 108.566}, "triton_wait": {"avg_ms": 50.025, "count": 129, "max_ms": 85.09, "p95_ms": 82.065}, "visualize": {"avg_ms": 3.689, "count": 129, "max_ms": 4.704, "p95_ms": 3.889}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.87, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2497}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4304.992, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T03:16:14+09:00", "last_error_summary": "[h264 @ 0x55e397acb7c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e397acb7c0] error while decoding MB 0 21, bytestream 1372 [h264 @ 0x55e396e29180] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e396e29180] error while decoding MB 0 30, bytestream 7259 [h264 @ 0x55e396e15140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e396e15140] error while decoding MB 0 4, bytestream 6801 [h264 @ 0x55e39792dac0] left block unavailable for reques", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 201, "h264_decode_error_count": 356, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:16:14+09:00", "last_error_summary": "[h264 @ 0x55e397acb7c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e397acb7c0] error while decoding MB 0 21, bytestream 1372 [h264 @ 0x55e396e29180] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e396e29180] error while decoding MB 0 30, bytestream 7259 [h264 @ 0x55e396e15140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e396e15140] error while decoding MB 0 4, bytestream 6801 [h264 @ 0x55e39792dac0] left block unavailable for reques", "last_exit_at": "2026-07-02T03:16:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 810.53, "last_started_at": "2026-07-02T03:02:43+09:00", "pid": 34421, "process": {"exists": false, "pid": 34421}, "restart_count": 226, "restart_count_per_hour": 5, "rtsp_timeout_count": 86, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 227}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 96.993, "sample_count": 2813, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 299645, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:16:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-02T03:16:14+09:00", "last_frame_read_age_sec": 0.108, "last_frame_read_at": "2026-07-02T03:16:14+09:00", "mjpeg": {"age_sec": 0.105, "exists": true, "mtime": "2026-07-02T03:16:14.258773+09:00", "mtime_epoch": 1782929774.2587733, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 112838}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T03:16:14+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1007.098, "name": "celery", "pid": 237, "rss_mb": 1005.035, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 130, "pushed_frames": 130, "read_frames": 130, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.345, "pushed": 4.345, "read": 4.345}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.531, "count": 130, "max_ms": 1.873, "p95_ms": 1.586}, "gstreamer_push": {"avg_ms": 1.52, "count": 130, "max_ms": 6.232, "p95_ms": 3.652}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.844, "count": 130, "max_ms": 75.871, "p95_ms": 53.653}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.407, "count": 130, "max_ms": 83.827, "p95_ms": 75.591}, "yolo11_640_fire_11class": {"avg_ms": 48.339, "count": 130, "max_ms": 77.506, "p95_ms": 73.551}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.983, "count": 130, "max_ms": 85.289, "p95_ms": 82.052}}, "truncated_count": 0}, "processing_total": {"avg_ms": 60.593, "count": 130, "max_ms": 96.338, "p95_ms": 92.891}, "read_interval": {"avg_ms": 237.793, "count": 130, "max_ms": 10124.532, "p95_ms": 108.566}, "triton_wait": {"avg_ms": 50.297, "count": 130, "max_ms": 85.303, "p95_ms": 82.648}, "visualize": {"avg_ms": 3.692, "count": 130, "max_ms": 4.704, "p95_ms": 3.895}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.922, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2497}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4305.012, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 201, "h264_decode_error_count": 356, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:16:14+09:00", "last_error_summary": "[h264 @ 0x55e397acb7c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e397acb7c0] error while decoding MB 0 21, bytestream 1372 [h264 @ 0x55e396e29180] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e396e29180] error while decoding MB 0 30, bytestream 7259 [h264 @ 0x55e396e15140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e396e15140] error while decoding MB 0 4, bytestream 6801 [h264 @ 0x55e39792dac0] left block unavailable for reques", "last_exit_at": "2026-07-02T03:16:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 810.539, "last_started_at": "2026-07-02T03:02:43+09:00", "pid": 34421, "process": {"exists": false, "pid": 34421}, "restart_count": 226, "restart_count_per_hour": 5, "rtsp_timeout_count": 86, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 227}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 96.993, "sample_count": 2813, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 299645, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:16:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.019, "last_frame_pushed_at": "2026-07-02T03:16:14+09:00", "last_frame_read_age_sec": 0.117, "last_frame_read_at": "2026-07-02T03:16:14+09:00", "mjpeg": {"age_sec": 0.114, "exists": true, "mtime": "2026-07-02T03:16:14.258773+09:00", "mtime_epoch": 1782929774.2587733, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 112838}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T03:16:14+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1007.098, "name": "celery", "pid": 237, "rss_mb": 1005.035, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 130, "pushed_frames": 130, "read_frames": 130, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.343, "pushed": 4.343, "read": 4.343}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.531, "count": 130, "max_ms": 1.873, "p95_ms": 1.586}, "gstreamer_push": {"avg_ms": 1.52, "count": 130, "max_ms": 6.232, "p95_ms": 3.652}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.844, "count": 130, "max_ms": 75.871, "p95_ms": 53.653}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.407, "count": 130, "max_ms": 83.827, "p95_ms": 75.591}, "yolo11_640_fire_11class": {"avg_ms": 48.339, "count": 130, "max_ms": 77.506, "p95_ms": 73.551}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.983, "count": 130, "max_ms": 85.289, "p95_ms": 82.052}}, "truncated_count": 0}, "processing_total": {"avg_ms": 60.593, "count": 130, "max_ms": 96.338, "p95_ms": 92.891}, "read_interval": {"avg_ms": 237.793, "count": 130, "max_ms": 10124.532, "p95_ms": 108.566}, "triton_wait": {"avg_ms": 50.297, "count": 130, "max_ms": 85.303, "p95_ms": 82.648}, "visualize": {"avg_ms": 3.692, "count": 130, "max_ms": 4.704, "p95_ms": 3.895}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.931, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2497}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4420.82, "limit_mb": 8796093022207.996}, "detail": {"pid": 35136, "restart_count": 227, "start_count": 228}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 201, "h264_decode_error_count": 356, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:16:14+09:00", "last_error_summary": "[h264 @ 0x55e397acb7c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e397acb7c0] error while decoding MB 0 21, bytestream 1372 [h264 @ 0x55e396e29180] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e396e29180] error while decoding MB 0 30, bytestream 7259 [h264 @ 0x55e396e15140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e396e15140] error while decoding MB 0 4, bytestream 6801 [h264 @ 0x55e39792dac0] left block unavailable for reques", "last_exit_at": "2026-07-02T03:16:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T03:16:16+09:00", "pid": 35136, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.047, "name": "ffmpeg", "pid": 35136, "rss_mb": 139.047, "state": "S (sleeping)", "threads": 106}, "restart_count": 227, "restart_count_per_hour": 6, "rtsp_timeout_count": 86, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 228}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 96.995, "sample_count": 2774, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 299645, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:16:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.296, "last_frame_pushed_at": "2026-07-02T03:16:14+09:00", "last_frame_read_age_sec": 4.394, "last_frame_read_at": "2026-07-02T03:16:14+09:00", "mjpeg": {"age_sec": 4.28, "exists": true, "mtime": "2026-07-02T03:16:14.370775+09:00", "mtime_epoch": 1782929774.370775, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 114915}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T03:16:18+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1007.098, "name": "celery", "pid": 237, "rss_mb": 1005.035, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 74, "pushed_frames": 74, "read_frames": 73, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.468, "pushed": 2.468, "read": 2.435}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.519, "count": 74, "max_ms": 1.873, "p95_ms": 1.586}, "gstreamer_push": {"avg_ms": 1.655, "count": 74, "max_ms": 6.232, "p95_ms": 3.884}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.289, "count": 74, "max_ms": 75.871, "p95_ms": 68.125}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 51.066, "count": 74, "max_ms": 83.827, "p95_ms": 76.594}, "yolo11_640_fire_11class": {"avg_ms": 49.83, "count": 74, "max_ms": 77.506, "p95_ms": 75.436}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.859, "count": 74, "max_ms": 85.289, "p95_ms": 83.315}}, "truncated_count": 0}, "processing_total": {"avg_ms": 62.63, "count": 74, "max_ms": 96.338, "p95_ms": 94.088}, "read_interval": {"avg_ms": 351.312, "count": 73, "max_ms": 10124.532, "p95_ms": 109.751}, "triton_wait": {"avg_ms": 52.252, "count": 74, "max_ms": 85.303, "p95_ms": 83.842}, "visualize": {"avg_ms": 3.727, "count": 74, "max_ms": 4.704, "p95_ms": 3.955}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.982, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3504}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4251.668, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x5597095ce200] missing picture in access unit with size 18 [h264 @ 0x559709c31540] no frame! [in#0/rtsp @ 0x5597095c9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5597095c9e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x5597095ce840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 55, "h264_decode_error_count": 56, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:19:46+09:00", "last_error_summary": "[NULL @ 0x5597095ce200] missing picture in access unit with size 18 [h264 @ 0x559709c31540] no frame! [in#0/rtsp @ 0x5597095c9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5597095c9e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x5597095ce840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T03:15:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 228.824, "last_started_at": "2026-07-02T03:15:57+09:00", "pid": 34984, "process": {"exists": false, "pid": 34984}, "restart_count": 54, "restart_count_per_hour": 10, "rtsp_timeout_count": 98, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 55}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.068, "sample_count": 2211, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 420550, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:19:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-02T03:19:46+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-02T03:19:46+09:00", "mjpeg": {"age_sec": 0.057, "exists": true, "mtime": "2026-07-02T03:19:46.097862+09:00", "mtime_epoch": 1782929986.0978625, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97648}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:19:46+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1065.324, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 26, "pushed_frames": 26, "read_frames": 26, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.867, "pushed": 0.867, "read": 0.867}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.419, "count": 27, "max_ms": 4.052, "p95_ms": 3.497}, "gstreamer_push": {"avg_ms": 1.435, "count": 26, "max_ms": 3.558, "p95_ms": 1.467}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.92, "count": 27, "max_ms": 74.547, "p95_ms": 73.305}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.402, "count": 27, "max_ms": 84.228, "p95_ms": 76.854}, "yolo11_640_fire_11class": {"avg_ms": 62.585, "count": 27, "max_ms": 76.367, "p95_ms": 75.394}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.978, "count": 27, "max_ms": 83.955, "p95_ms": 83.749}}, "truncated_count": 0}, "processing_total": {"avg_ms": 81.076, "count": 26, "max_ms": 96.522, "p95_ms": 96.479}, "read_interval": {"avg_ms": 1154.418, "count": 26, "max_ms": 16276.848, "p95_ms": 11429.645}, "triton_wait": {"avg_ms": 68.224, "count": 27, "max_ms": 84.238, "p95_ms": 83.967}, "visualize": {"avg_ms": 3.635, "count": 27, "max_ms": 3.861, "p95_ms": 3.846}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.987, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3504}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4251.504, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T03:19:46+09:00", "last_error_summary": "[NULL @ 0x5597095ce200] missing picture in access unit with size 18 [h264 @ 0x559709c31540] no frame! [in#0/rtsp @ 0x5597095c9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5597095c9e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x5597095ce840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 55, "h264_decode_error_count": 56, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:19:46+09:00", "last_error_summary": "[NULL @ 0x5597095ce200] missing picture in access unit with size 18 [h264 @ 0x559709c31540] no frame! [in#0/rtsp @ 0x5597095c9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5597095c9e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x5597095ce840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T03:19:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 228.838, "last_started_at": "2026-07-02T03:15:57+09:00", "pid": 34984, "process": {"exists": false, "pid": 34984}, "restart_count": 54, "restart_count_per_hour": 10, "rtsp_timeout_count": 98, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 55}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.068, "sample_count": 2212, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 420550, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:19:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T03:19:46+09:00", "last_frame_read_age_sec": 0.079, "last_frame_read_at": "2026-07-02T03:19:46+09:00", "mjpeg": {"age_sec": 0.071, "exists": true, "mtime": "2026-07-02T03:19:46.097862+09:00", "mtime_epoch": 1782929986.0978625, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97648}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:19:46+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1065.324, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 26, "pushed_frames": 27, "read_frames": 26, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.867, "pushed": 0.9, "read": 0.867}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.42, "count": 26, "max_ms": 4.052, "p95_ms": 3.497}, "gstreamer_push": {"avg_ms": 1.421, "count": 27, "max_ms": 3.558, "p95_ms": 1.467}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.598, "count": 26, "max_ms": 74.547, "p95_ms": 73.305}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.075, "count": 26, "max_ms": 84.228, "p95_ms": 76.854}, "yolo11_640_fire_11class": {"avg_ms": 62.187, "count": 26, "max_ms": 76.367, "p95_ms": 75.394}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.459, "count": 26, "max_ms": 83.955, "p95_ms": 83.749}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.215, "count": 26, "max_ms": 96.522, "p95_ms": 96.479}, "read_interval": {"avg_ms": 1154.418, "count": 26, "max_ms": 16276.848, "p95_ms": 11429.645}, "triton_wait": {"avg_ms": 67.752, "count": 26, "max_ms": 84.238, "p95_ms": 83.967}, "visualize": {"avg_ms": 3.635, "count": 26, "max_ms": 3.861, "p95_ms": 3.846}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3504}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4251.305, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 55, "h264_decode_error_count": 56, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:19:46+09:00", "last_error_summary": "[NULL @ 0x5597095ce200] missing picture in access unit with size 18 [h264 @ 0x559709c31540] no frame! [in#0/rtsp @ 0x5597095c9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5597095c9e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x5597095ce840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T03:19:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 228.843, "last_started_at": "2026-07-02T03:15:57+09:00", "pid": 34984, "process": {"exists": false, "pid": 34984}, "restart_count": 54, "restart_count_per_hour": 10, "rtsp_timeout_count": 98, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 55}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.068, "sample_count": 2212, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 420550, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:19:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-02T03:19:46+09:00", "last_frame_read_age_sec": 0.085, "last_frame_read_at": "2026-07-02T03:19:46+09:00", "mjpeg": {"age_sec": 0.077, "exists": true, "mtime": "2026-07-02T03:19:46.097862+09:00", "mtime_epoch": 1782929986.0978625, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97648}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:19:46+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1065.324, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 26, "pushed_frames": 26, "read_frames": 26, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.867, "pushed": 0.867, "read": 0.867}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.42, "count": 26, "max_ms": 4.052, "p95_ms": 3.497}, "gstreamer_push": {"avg_ms": 1.422, "count": 26, "max_ms": 3.558, "p95_ms": 1.467}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.598, "count": 26, "max_ms": 74.547, "p95_ms": 73.305}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.075, "count": 26, "max_ms": 84.228, "p95_ms": 76.854}, "yolo11_640_fire_11class": {"avg_ms": 62.187, "count": 26, "max_ms": 76.367, "p95_ms": 75.394}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.459, "count": 26, "max_ms": 83.955, "p95_ms": 83.749}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.215, "count": 26, "max_ms": 96.522, "p95_ms": 96.479}, "read_interval": {"avg_ms": 1154.418, "count": 26, "max_ms": 16276.848, "p95_ms": 11429.645}, "triton_wait": {"avg_ms": 67.752, "count": 26, "max_ms": 84.238, "p95_ms": 83.967}, "visualize": {"avg_ms": 3.635, "count": 26, "max_ms": 3.861, "p95_ms": 3.846}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3504}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4325.762, "limit_mb": 8796093022207.996}, "detail": {"pid": 35243, "restart_count": 55, "start_count": 56}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 55, "h264_decode_error_count": 56, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:19:46+09:00", "last_error_summary": "[NULL @ 0x5597095ce200] missing picture in access unit with size 18 [h264 @ 0x559709c31540] no frame! [in#0/rtsp @ 0x5597095c9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5597095c9e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x5597095ce840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T03:19:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T03:19:48+09:00", "pid": 35243, "process": {"exists": true, "fd_count": 5, "hwm_mb": 107.754, "name": "ffmpeg", "pid": 35243, "rss_mb": 107.754, "state": "S (sleeping)", "threads": 19}, "restart_count": 55, "restart_count_per_hour": 11, "rtsp_timeout_count": 98, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 56}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.119, "sample_count": 2170, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 420550, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:19:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.654, "last_frame_pushed_at": "2026-07-02T03:19:46+09:00", "last_frame_read_age_sec": 4.726, "last_frame_read_at": "2026-07-02T03:19:46+09:00", "mjpeg": {"age_sec": 4.638, "exists": true, "mtime": "2026-07-02T03:19:46.177864+09:00", "mtime_epoch": 1782929986.1778636, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 99251}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:19:50+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1065.324, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.594, "pushed": 0.594, "read": 0.594}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.397, "count": 13, "max_ms": 3.497, "p95_ms": 3.497}, "gstreamer_push": {"avg_ms": 1.471, "count": 13, "max_ms": 3.558, "p95_ms": 3.558}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 41.825, "count": 13, "max_ms": 50.784, "p95_ms": 50.784}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.801, "count": 13, "max_ms": 62.174, "p95_ms": 62.174}, "yolo11_640_fire_11class": {"avg_ms": 50.66, "count": 13, "max_ms": 56.55, "p95_ms": 56.55}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.442, "count": 13, "max_ms": 57.719, "p95_ms": 57.719}}, "truncated_count": 0}, "processing_total": {"avg_ms": 66.494, "count": 13, "max_ms": 74.215, "p95_ms": 74.215}, "read_interval": {"avg_ms": 2198.97, "count": 13, "max_ms": 16276.848, "p95_ms": 16276.848}, "triton_wait": {"avg_ms": 53.93, "count": 13, "max_ms": 62.186, "p95_ms": 62.186}, "visualize": {"avg_ms": 3.57, "count": 13, "max_ms": 3.754, "p95_ms": 3.754}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.883, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3520}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4264.527, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5653c0d66140] error while decoding MB 4 47, bytestream -5 [in#0/rtsp @ 0x5653c0befe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5653c0befe00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 56, "h264_decode_error_count": 57, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:23:47+09:00", "last_error_summary": "[h264 @ 0x5653c0d66140] error while decoding MB 4 47, bytestream -5 [in#0/rtsp @ 0x5653c0befe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5653c0befe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:19:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 238.185, "last_started_at": "2026-07-02T03:19:48+09:00", "pid": 35243, "process": {"exists": false, "pid": 35243}, "restart_count": 55, "restart_count_per_hour": 11, "rtsp_timeout_count": 100, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 56}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.986, "sample_count": 2235, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 422494, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:23:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.078, "last_frame_pushed_at": "2026-07-02T03:23:46+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-02T03:23:46+09:00", "mjpeg": {"age_sec": 0.063, "exists": true, "mtime": "2026-07-02T03:23:46.937471+09:00", "mtime_epoch": 1782930226.9374707, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 100196}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:23:47+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1068.852, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 76, "pushed_frames": 76, "read_frames": 76, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.534, "pushed": 2.534, "read": 2.534}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.422, "count": 77, "max_ms": 5.182, "p95_ms": 3.481}, "gstreamer_push": {"avg_ms": 1.63, "count": 76, "max_ms": 6.243, "p95_ms": 3.858}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 64.093, "count": 77, "max_ms": 75.312, "p95_ms": 74.11}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.371, "count": 77, "max_ms": 80.753, "p95_ms": 77.605}, "yolo11_640_fire_11class": {"avg_ms": 69.596, "count": 77, "max_ms": 78.325, "p95_ms": 77.216}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.887, "count": 77, "max_ms": 85.92, "p95_ms": 84.619}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.832, "count": 76, "max_ms": 97.972, "p95_ms": 96.875}, "read_interval": {"avg_ms": 394.914, "count": 76, "max_ms": 22046.068, "p95_ms": 114.462}, "triton_wait": {"avg_ms": 77.215, "count": 77, "max_ms": 85.931, "p95_ms": 84.631}, "visualize": {"avg_ms": 3.687, "count": 77, "max_ms": 4.709, "p95_ms": 3.864}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3520}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4264.109, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T03:23:47+09:00", "last_error_summary": "[h264 @ 0x5653c0d66140] error while decoding MB 4 47, bytestream -5 [in#0/rtsp @ 0x5653c0befe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5653c0befe00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 56, "h264_decode_error_count": 57, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:23:47+09:00", "last_error_summary": "[h264 @ 0x5653c0d66140] error while decoding MB 4 47, bytestream -5 [in#0/rtsp @ 0x5653c0befe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5653c0befe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:23:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 238.205, "last_started_at": "2026-07-02T03:19:48+09:00", "pid": 35243, "process": {"exists": false, "pid": 35243}, "restart_count": 55, "restart_count_per_hour": 11, "rtsp_timeout_count": 100, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 56}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.986, "sample_count": 2236, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 422494, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:23:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-02T03:23:47+09:00", "last_frame_read_age_sec": 0.085, "last_frame_read_at": "2026-07-02T03:23:46+09:00", "mjpeg": {"age_sec": 0.083, "exists": true, "mtime": "2026-07-02T03:23:46.937471+09:00", "mtime_epoch": 1782930226.9374707, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 100196}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:23:47+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1068.852, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 76, "pushed_frames": 76, "read_frames": 76, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.534, "pushed": 2.534, "read": 2.534}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.422, "count": 76, "max_ms": 5.182, "p95_ms": 3.481}, "gstreamer_push": {"avg_ms": 1.653, "count": 76, "max_ms": 6.243, "p95_ms": 3.858}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 64.086, "count": 76, "max_ms": 75.312, "p95_ms": 74.11}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.323, "count": 76, "max_ms": 80.753, "p95_ms": 77.605}, "yolo11_640_fire_11class": {"avg_ms": 69.626, "count": 76, "max_ms": 78.325, "p95_ms": 77.216}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.899, "count": 76, "max_ms": 85.92, "p95_ms": 84.619}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.597, "count": 76, "max_ms": 97.972, "p95_ms": 96.875}, "read_interval": {"avg_ms": 394.914, "count": 76, "max_ms": 22046.068, "p95_ms": 114.462}, "triton_wait": {"avg_ms": 77.231, "count": 76, "max_ms": 85.931, "p95_ms": 84.631}, "visualize": {"avg_ms": 3.687, "count": 76, "max_ms": 4.709, "p95_ms": 3.864}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3520}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4264.113, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 56, "h264_decode_error_count": 57, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:23:47+09:00", "last_error_summary": "[h264 @ 0x5653c0d66140] error while decoding MB 4 47, bytestream -5 [in#0/rtsp @ 0x5653c0befe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5653c0befe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:23:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 238.213, "last_started_at": "2026-07-02T03:19:48+09:00", "pid": 35243, "process": {"exists": false, "pid": 35243}, "restart_count": 55, "restart_count_per_hour": 11, "rtsp_timeout_count": 100, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 56}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.986, "sample_count": 2236, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 422494, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:23:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.018, "last_frame_pushed_at": "2026-07-02T03:23:47+09:00", "last_frame_read_age_sec": 0.093, "last_frame_read_at": "2026-07-02T03:23:46+09:00", "mjpeg": {"age_sec": 0.007, "exists": true, "mtime": "2026-07-02T03:23:47.021472+09:00", "mtime_epoch": 1782930227.021472, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 100956}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:23:47+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1068.852, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 76, "pushed_frames": 76, "read_frames": 76, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.533, "pushed": 2.533, "read": 2.533}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.422, "count": 76, "max_ms": 5.182, "p95_ms": 3.481}, "gstreamer_push": {"avg_ms": 1.653, "count": 76, "max_ms": 6.243, "p95_ms": 3.858}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 64.086, "count": 76, "max_ms": 75.312, "p95_ms": 74.11}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.323, "count": 76, "max_ms": 80.753, "p95_ms": 77.605}, "yolo11_640_fire_11class": {"avg_ms": 69.626, "count": 76, "max_ms": 78.325, "p95_ms": 77.216}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.899, "count": 76, "max_ms": 85.92, "p95_ms": 84.619}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.597, "count": 76, "max_ms": 97.972, "p95_ms": 96.875}, "read_interval": {"avg_ms": 394.914, "count": 76, "max_ms": 22046.068, "p95_ms": 114.462}, "triton_wait": {"avg_ms": 77.231, "count": 76, "max_ms": 85.931, "p95_ms": 84.631}, "visualize": {"avg_ms": 3.687, "count": 76, "max_ms": 4.709, "p95_ms": 3.864}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3520}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4284.402, "limit_mb": 8796093022207.996}, "detail": {"pid": 35395, "restart_count": 56, "start_count": 57}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 56, "h264_decode_error_count": 57, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:23:47+09:00", "last_error_summary": "[h264 @ 0x5653c0d66140] error while decoding MB 4 47, bytestream -5 [in#0/rtsp @ 0x5653c0befe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5653c0befe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:23:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T03:23:49+09:00", "pid": 35395, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.242, "name": "ffmpeg", "pid": 35395, "rss_mb": 58.242, "state": "S (sleeping)", "threads": 1}, "restart_count": 56, "restart_count_per_hour": 12, "rtsp_timeout_count": 100, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": true, "start_count": 57}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.987, "sample_count": 2194, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 422494, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:23:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.627, "last_frame_pushed_at": "2026-07-02T03:23:47+09:00", "last_frame_read_age_sec": 4.701, "last_frame_read_at": "2026-07-02T03:23:46+09:00", "mjpeg": {"age_sec": 4.615, "exists": true, "mtime": "2026-07-02T03:23:47.021472+09:00", "mtime_epoch": 1782930227.021472, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 100956}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:23:51+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1068.852, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 35, "pushed_frames": 35, "read_frames": 34, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.168, "pushed": 1.168, "read": 1.134}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.452, "count": 35, "max_ms": 5.182, "p95_ms": 4.086}, "gstreamer_push": {"avg_ms": 1.809, "count": 35, "max_ms": 6.243, "p95_ms": 3.858}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.258, "count": 35, "max_ms": 74.237, "p95_ms": 74.11}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.793, "count": 35, "max_ms": 78.432, "p95_ms": 78.344}, "yolo11_640_fire_11class": {"avg_ms": 66.047, "count": 35, "max_ms": 78.325, "p95_ms": 77.273}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.04, "count": 35, "max_ms": 84.302, "p95_ms": 84.123}}, "truncated_count": 0}, "processing_total": {"avg_ms": 84.248, "count": 35, "max_ms": 96.418, "p95_ms": 96.364}, "read_interval": {"avg_ms": 746.085, "count": 34, "max_ms": 22046.068, "p95_ms": 114.462}, "triton_wait": {"avg_ms": 71.747, "count": 35, "max_ms": 84.314, "p95_ms": 84.135}, "visualize": {"avg_ms": 3.716, "count": 35, "max_ms": 4.709, "p95_ms": 4.026}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.974, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3541}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4284.754, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x562251375480] error while decoding MB 52 21, bytestream -5 [in#0/rtsp @ 0x5622509d0e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5622509d0e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 57, "h264_decode_error_count": 58, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:28:44+09:00", "last_error_summary": "[h264 @ 0x562251375480] error while decoding MB 52 21, bytestream -5 [in#0/rtsp @ 0x5622509d0e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5622509d0e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:23:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 294.373, "last_started_at": "2026-07-02T03:23:49+09:00", "pid": 35395, "process": {"exists": false, "pid": 35395}, "restart_count": 56, "restart_count_per_hour": 11, "rtsp_timeout_count": 102, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 57}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.383, "sample_count": 2501, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 424983, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:28:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.076, "last_frame_pushed_at": "2026-07-02T03:28:43+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-02T03:28:43+09:00", "mjpeg": {"age_sec": 0.061, "exists": true, "mtime": "2026-07-02T03:28:43.947249+09:00", "mtime_epoch": 1782930523.9472492, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 98706}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:28:44+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1067.305, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 109, "pushed_frames": 109, "read_frames": 110, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.209, "pushed": 6.209, "read": 6.266}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.466, "count": 110, "max_ms": 5.085, "p95_ms": 4.114}, "gstreamer_push": {"avg_ms": 1.508, "count": 109, "max_ms": 4.645, "p95_ms": 3.76}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.707, "count": 110, "max_ms": 80.781, "p95_ms": 70.733}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.738, "count": 110, "max_ms": 81.353, "p95_ms": 78.795}, "yolo11_640_fire_11class": {"avg_ms": 60.227, "count": 110, "max_ms": 80.769, "p95_ms": 78.161}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.553, "count": 110, "max_ms": 89.162, "p95_ms": 84.746}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.501, "count": 109, "max_ms": 102.072, "p95_ms": 96.855}, "read_interval": {"avg_ms": 310.065, "count": 110, "max_ms": 16617.67, "p95_ms": 114.165}, "triton_wait": {"avg_ms": 65.094, "count": 110, "max_ms": 89.175, "p95_ms": 84.76}, "visualize": {"avg_ms": 3.7, "count": 110, "max_ms": 4.733, "p95_ms": 4.129}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.554, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3541}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4284.574, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T03:28:44+09:00", "last_error_summary": "[h264 @ 0x562251375480] error while decoding MB 52 21, bytestream -5 [in#0/rtsp @ 0x5622509d0e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5622509d0e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 57, "h264_decode_error_count": 58, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:28:44+09:00", "last_error_summary": "[h264 @ 0x562251375480] error while decoding MB 52 21, bytestream -5 [in#0/rtsp @ 0x5622509d0e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5622509d0e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:28:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 294.393, "last_started_at": "2026-07-02T03:23:49+09:00", "pid": 35395, "process": {"exists": false, "pid": 35395}, "restart_count": 56, "restart_count_per_hour": 11, "rtsp_timeout_count": 102, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 57}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.383, "sample_count": 2502, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 424983, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:28:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T03:28:44+09:00", "last_frame_read_age_sec": 0.084, "last_frame_read_at": "2026-07-02T03:28:43+09:00", "mjpeg": {"age_sec": 0.081, "exists": true, "mtime": "2026-07-02T03:28:43.947249+09:00", "mtime_epoch": 1782930523.9472492, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 98706}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:28:44+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1067.305, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 110, "pushed_frames": 110, "read_frames": 110, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.26, "pushed": 6.26, "read": 6.26}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.466, "count": 110, "max_ms": 5.085, "p95_ms": 4.114}, "gstreamer_push": {"avg_ms": 1.506, "count": 110, "max_ms": 4.645, "p95_ms": 3.76}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.707, "count": 110, "max_ms": 80.781, "p95_ms": 70.733}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.738, "count": 110, "max_ms": 81.353, "p95_ms": 78.795}, "yolo11_640_fire_11class": {"avg_ms": 60.227, "count": 110, "max_ms": 80.769, "p95_ms": 78.161}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.553, "count": 110, "max_ms": 89.162, "p95_ms": 84.746}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.474, "count": 110, "max_ms": 102.072, "p95_ms": 96.855}, "read_interval": {"avg_ms": 310.065, "count": 110, "max_ms": 16617.67, "p95_ms": 114.165}, "triton_wait": {"avg_ms": 65.094, "count": 110, "max_ms": 89.175, "p95_ms": 84.76}, "visualize": {"avg_ms": 3.7, "count": 110, "max_ms": 4.733, "p95_ms": 4.129}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.573, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3541}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4284.281, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 57, "h264_decode_error_count": 58, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:28:44+09:00", "last_error_summary": "[h264 @ 0x562251375480] error while decoding MB 52 21, bytestream -5 [in#0/rtsp @ 0x5622509d0e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5622509d0e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:28:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 294.401, "last_started_at": "2026-07-02T03:23:49+09:00", "pid": 35395, "process": {"exists": false, "pid": 35395}, "restart_count": 56, "restart_count_per_hour": 11, "rtsp_timeout_count": 102, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 57}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.383, "sample_count": 2502, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 424983, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:28:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-02T03:28:44+09:00", "last_frame_read_age_sec": 0.092, "last_frame_read_at": "2026-07-02T03:28:43+09:00", "mjpeg": {"age_sec": 0.09, "exists": true, "mtime": "2026-07-02T03:28:43.947249+09:00", "mtime_epoch": 1782930523.9472492, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 98706}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:28:44+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1067.305, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 110, "pushed_frames": 110, "read_frames": 110, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.256, "pushed": 6.256, "read": 6.256}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.466, "count": 110, "max_ms": 5.085, "p95_ms": 4.114}, "gstreamer_push": {"avg_ms": 1.506, "count": 110, "max_ms": 4.645, "p95_ms": 3.76}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.707, "count": 110, "max_ms": 80.781, "p95_ms": 70.733}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.738, "count": 110, "max_ms": 81.353, "p95_ms": 78.795}, "yolo11_640_fire_11class": {"avg_ms": 60.227, "count": 110, "max_ms": 80.769, "p95_ms": 78.161}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.553, "count": 110, "max_ms": 89.162, "p95_ms": 84.746}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.474, "count": 110, "max_ms": 102.072, "p95_ms": 96.855}, "read_interval": {"avg_ms": 310.065, "count": 110, "max_ms": 16617.67, "p95_ms": 114.165}, "triton_wait": {"avg_ms": 65.094, "count": 110, "max_ms": 89.175, "p95_ms": 84.76}, "visualize": {"avg_ms": 3.7, "count": 110, "max_ms": 4.733, "p95_ms": 4.129}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.582, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3541}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4413.539, "limit_mb": 8796093022207.996}, "detail": {"pid": 35547, "restart_count": 57, "start_count": 58}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 57, "h264_decode_error_count": 58, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:28:44+09:00", "last_error_summary": "[h264 @ 0x562251375480] error while decoding MB 52 21, bytestream -5 [in#0/rtsp @ 0x5622509d0e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5622509d0e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:28:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T03:28:46+09:00", "pid": 35547, "process": {"exists": true, "fd_count": 5, "hwm_mb": 156.609, "name": "ffmpeg", "pid": 35547, "rss_mb": 156.609, "state": "R (running)", "threads": 49}, "restart_count": 57, "restart_count_per_hour": 12, "rtsp_timeout_count": 102, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": true, "start_count": 58}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.386, "sample_count": 2489, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 424983, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:28:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.907, "last_frame_pushed_at": "2026-07-02T03:28:44+09:00", "last_frame_read_age_sec": 4.983, "last_frame_read_at": "2026-07-02T03:28:43+09:00", "mjpeg": {"age_sec": 4.889, "exists": true, "mtime": "2026-07-02T03:28:44.039253+09:00", "mtime_epoch": 1782930524.0392535, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 98943}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:28:48+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1067.305, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 110, "pushed_frames": 110, "read_frames": 110, "select_timeouts": 15, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.895, "pushed": 4.895, "read": 4.895}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.466, "count": 110, "max_ms": 5.085, "p95_ms": 4.114}, "gstreamer_push": {"avg_ms": 1.506, "count": 110, "max_ms": 4.645, "p95_ms": 3.76}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.707, "count": 110, "max_ms": 80.781, "p95_ms": 70.733}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.738, "count": 110, "max_ms": 81.353, "p95_ms": 78.795}, "yolo11_640_fire_11class": {"avg_ms": 60.227, "count": 110, "max_ms": 80.769, "p95_ms": 78.161}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.553, "count": 110, "max_ms": 89.162, "p95_ms": 84.746}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.474, "count": 110, "max_ms": 102.072, "p95_ms": 96.855}, "read_interval": {"avg_ms": 310.065, "count": 110, "max_ms": 16617.67, "p95_ms": 114.165}, "triton_wait": {"avg_ms": 65.094, "count": 110, "max_ms": 89.175, "p95_ms": 84.76}, "visualize": {"avg_ms": 3.7, "count": 110, "max_ms": 4.733, "p95_ms": 4.129}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.473, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2558}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4313.742, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 9, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x557b912bba80] top block unavailable for requested intra mode [h264 @ 0x557b912bba80] error while decoding MB 57 0, bytestream 1107 [h264 @ 0x557b912cfc40] error while decoding MB 38 27, bytestream -5 [h264 @ 0x557b911fc080] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557b911fc080] error while decoding MB 0 31, bytestream 36731 [h264 @ 0x557b911fc080] cabac decode of qscale diff failed at 52 24 [h264 @ 0x557b911fc080] error while decoding MB 52 24, bytestream 14579 [", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 202, "h264_decode_error_count": 365, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:29:20+09:00", "last_error_summary": "[h264 @ 0x557b912bba80] top block unavailable for requested intra mode [h264 @ 0x557b912bba80] error while decoding MB 57 0, bytestream 1107 [h264 @ 0x557b912cfc40] error while decoding MB 38 27, bytestream -5 [h264 @ 0x557b911fc080] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557b911fc080] error while decoding MB 0 31, bytestream 36731 [h264 @ 0x557b911fc080] cabac decode of qscale diff failed at 52 24 [h264 @ 0x557b911fc080] error while decoding MB 52 24, bytestream 14579 [", "last_exit_at": "2026-07-02T03:16:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 784.31, "last_started_at": "2026-07-02T03:16:16+09:00", "pid": 35136, "process": {"exists": false, "pid": 35136}, "restart_count": 227, "restart_count_per_hour": 5, "rtsp_timeout_count": 88, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 228}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.357, "sample_count": 2697, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 306982, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:29:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.086, "last_frame_pushed_at": "2026-07-02T03:29:20+09:00", "last_frame_read_age_sec": 0.074, "last_frame_read_at": "2026-07-02T03:29:20+09:00", "mjpeg": {"age_sec": 0.071, "exists": true, "mtime": "2026-07-02T03:29:20.888945+09:00", "mtime_epoch": 1782930560.8889446, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 122712}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T03:29:20+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1008.902, "name": "celery", "pid": 237, "rss_mb": 1006.066, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 104, "pushed_frames": 104, "read_frames": 105, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.468, "pushed": 3.468, "read": 3.501}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.527, "count": 104, "max_ms": 1.905, "p95_ms": 1.609}, "gstreamer_push": {"avg_ms": 1.523, "count": 104, "max_ms": 6.062, "p95_ms": 1.624}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.3, "count": 104, "max_ms": 77.085, "p95_ms": 70.384}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.585, "count": 104, "max_ms": 81.585, "p95_ms": 79.791}, "yolo11_640_fire_11class": {"avg_ms": 63.418, "count": 104, "max_ms": 77.074, "p95_ms": 76.158}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.169, "count": 104, "max_ms": 86.846, "p95_ms": 85.365}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.607, "count": 104, "max_ms": 97.401, "p95_ms": 95.859}, "read_interval": {"avg_ms": 285.713, "count": 105, "max_ms": 20067.733, "p95_ms": 112.156}, "triton_wait": {"avg_ms": 69.208, "count": 104, "max_ms": 86.858, "p95_ms": 85.377}, "visualize": {"avg_ms": 3.686, "count": 104, "max_ms": 5.03, "p95_ms": 3.803}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.989, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2558}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4313.555, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T03:29:20+09:00", "last_error_summary": "[h264 @ 0x557b912bba80] top block unavailable for requested intra mode [h264 @ 0x557b912bba80] error while decoding MB 57 0, bytestream 1107 [h264 @ 0x557b912cfc40] error while decoding MB 38 27, bytestream -5 [h264 @ 0x557b911fc080] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557b911fc080] error while decoding MB 0 31, bytestream 36731 [h264 @ 0x557b911fc080] cabac decode of qscale diff failed at 52 24 [h264 @ 0x557b911fc080] error while decoding MB 52 24, bytestream 14579 [", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 202, "h264_decode_error_count": 365, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:29:20+09:00", "last_error_summary": "[h264 @ 0x557b912bba80] top block unavailable for requested intra mode [h264 @ 0x557b912bba80] error while decoding MB 57 0, bytestream 1107 [h264 @ 0x557b912cfc40] error while decoding MB 38 27, bytestream -5 [h264 @ 0x557b911fc080] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557b911fc080] error while decoding MB 0 31, bytestream 36731 [h264 @ 0x557b911fc080] cabac decode of qscale diff failed at 52 24 [h264 @ 0x557b911fc080] error while decoding MB 52 24, bytestream 14579 [", "last_exit_at": "2026-07-02T03:29:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 784.337, "last_started_at": "2026-07-02T03:16:16+09:00", "pid": 35136, "process": {"exists": false, "pid": 35136}, "restart_count": 227, "restart_count_per_hour": 5, "rtsp_timeout_count": 88, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 228}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.357, "sample_count": 2698, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 306983, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:29:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T03:29:20+09:00", "last_frame_read_age_sec": 0.1, "last_frame_read_at": "2026-07-02T03:29:20+09:00", "mjpeg": {"age_sec": 0.097, "exists": true, "mtime": "2026-07-02T03:29:20.888945+09:00", "mtime_epoch": 1782930560.8889446, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 122712}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T03:29:20+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1008.902, "name": "celery", "pid": 237, "rss_mb": 1006.066, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 105, "pushed_frames": 105, "read_frames": 104, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.506, "pushed": 3.506, "read": 3.473}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.527, "count": 105, "max_ms": 1.905, "p95_ms": 1.609}, "gstreamer_push": {"avg_ms": 1.521, "count": 105, "max_ms": 6.062, "p95_ms": 1.624}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.256, "count": 105, "max_ms": 77.085, "p95_ms": 70.384}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.736, "count": 105, "max_ms": 81.585, "p95_ms": 79.816}, "yolo11_640_fire_11class": {"avg_ms": 63.543, "count": 105, "max_ms": 77.074, "p95_ms": 76.344}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.289, "count": 105, "max_ms": 86.846, "p95_ms": 85.365}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.723, "count": 105, "max_ms": 97.401, "p95_ms": 95.859}, "read_interval": {"avg_ms": 287.644, "count": 104, "max_ms": 20067.733, "p95_ms": 112.156}, "triton_wait": {"avg_ms": 69.324, "count": 105, "max_ms": 86.858, "p95_ms": 85.377}, "visualize": {"avg_ms": 3.687, "count": 105, "max_ms": 5.03, "p95_ms": 3.803}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.948, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2558}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4313.387, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 202, "h264_decode_error_count": 365, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:29:20+09:00", "last_error_summary": "[h264 @ 0x557b912bba80] top block unavailable for requested intra mode [h264 @ 0x557b912bba80] error while decoding MB 57 0, bytestream 1107 [h264 @ 0x557b912cfc40] error while decoding MB 38 27, bytestream -5 [h264 @ 0x557b911fc080] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557b911fc080] error while decoding MB 0 31, bytestream 36731 [h264 @ 0x557b911fc080] cabac decode of qscale diff failed at 52 24 [h264 @ 0x557b911fc080] error while decoding MB 52 24, bytestream 14579 [", "last_exit_at": "2026-07-02T03:29:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 784.345, "last_started_at": "2026-07-02T03:16:16+09:00", "pid": 35136, "process": {"exists": false, "pid": 35136}, "restart_count": 227, "restart_count_per_hour": 5, "rtsp_timeout_count": 88, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 228}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.357, "sample_count": 2698, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 306983, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:29:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-02T03:29:20+09:00", "last_frame_read_age_sec": 0.109, "last_frame_read_at": "2026-07-02T03:29:20+09:00", "mjpeg": {"age_sec": 0.106, "exists": true, "mtime": "2026-07-02T03:29:20.888945+09:00", "mtime_epoch": 1782930560.8889446, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 122712}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T03:29:20+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1008.902, "name": "celery", "pid": 237, "rss_mb": 1006.066, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 105, "pushed_frames": 105, "read_frames": 104, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.505, "pushed": 3.505, "read": 3.472}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.527, "count": 105, "max_ms": 1.905, "p95_ms": 1.609}, "gstreamer_push": {"avg_ms": 1.521, "count": 105, "max_ms": 6.062, "p95_ms": 1.624}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.256, "count": 105, "max_ms": 77.085, "p95_ms": 70.384}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.736, "count": 105, "max_ms": 81.585, "p95_ms": 79.816}, "yolo11_640_fire_11class": {"avg_ms": 63.543, "count": 105, "max_ms": 77.074, "p95_ms": 76.344}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.289, "count": 105, "max_ms": 86.846, "p95_ms": 85.365}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.723, "count": 105, "max_ms": 97.401, "p95_ms": 95.859}, "read_interval": {"avg_ms": 287.644, "count": 104, "max_ms": 20067.733, "p95_ms": 112.156}, "triton_wait": {"avg_ms": 69.324, "count": 105, "max_ms": 86.858, "p95_ms": 85.377}, "visualize": {"avg_ms": 3.687, "count": 105, "max_ms": 5.03, "p95_ms": 3.803}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.957, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2558}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4420.715, "limit_mb": 8796093022207.996}, "detail": {"pid": 35699, "restart_count": 228, "start_count": 229}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 202, "h264_decode_error_count": 365, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:29:20+09:00", "last_error_summary": "[h264 @ 0x557b912bba80] top block unavailable for requested intra mode [h264 @ 0x557b912bba80] error while decoding MB 57 0, bytestream 1107 [h264 @ 0x557b912cfc40] error while decoding MB 38 27, bytestream -5 [h264 @ 0x557b911fc080] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557b911fc080] error while decoding MB 0 31, bytestream 36731 [h264 @ 0x557b911fc080] cabac decode of qscale diff failed at 52 24 [h264 @ 0x557b911fc080] error while decoding MB 52 24, bytestream 14579 [", "last_exit_at": "2026-07-02T03:29:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T03:29:23+09:00", "pid": 35699, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.297, "name": "ffmpeg", "pid": 35699, "rss_mb": 139.297, "state": "S (sleeping)", "threads": 106}, "restart_count": 228, "restart_count_per_hour": 6, "rtsp_timeout_count": 88, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 229}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.357, "sample_count": 2676, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 306983, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:29:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.096, "last_frame_pushed_at": "2026-07-02T03:29:20+09:00", "last_frame_read_age_sec": 4.19, "last_frame_read_at": "2026-07-02T03:29:20+09:00", "mjpeg": {"age_sec": 4.079, "exists": true, "mtime": "2026-07-02T03:29:20.996949+09:00", "mtime_epoch": 1782930560.9969494, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 124515}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T03:29:25+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1008.902, "name": "celery", "pid": 237, "rss_mb": 1006.066, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 63, "pushed_frames": 63, "read_frames": 62, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.106, "pushed": 2.106, "read": 2.073}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.522, "count": 63, "max_ms": 1.905, "p95_ms": 1.575}, "gstreamer_push": {"avg_ms": 1.435, "count": 63, "max_ms": 3.936, "p95_ms": 1.483}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.856, "count": 63, "max_ms": 76.977, "p95_ms": 66.513}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.326, "count": 63, "max_ms": 81.585, "p95_ms": 79.845}, "yolo11_640_fire_11class": {"avg_ms": 63.153, "count": 63, "max_ms": 76.743, "p95_ms": 76.496}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.319, "count": 63, "max_ms": 86.846, "p95_ms": 85.596}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.33, "count": 63, "max_ms": 97.401, "p95_ms": 95.971}, "read_interval": {"avg_ms": 416.324, "count": 62, "max_ms": 20067.733, "p95_ms": 111.243}, "triton_wait": {"avg_ms": 68.885, "count": 63, "max_ms": 86.858, "p95_ms": 85.608}, "visualize": {"avg_ms": 3.695, "count": 63, "max_ms": 3.871, "p95_ms": 3.803}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.908, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3554}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4265.203, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x563e5779b200] missing picture in access unit with size 18 [h264 @ 0x563e577d8600] no frame! [in#0/rtsp @ 0x563e57796e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563e57796e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x563e5779b840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 58, "h264_decode_error_count": 59, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:32:06+09:00", "last_error_summary": "[NULL @ 0x563e5779b200] missing picture in access unit with size 18 [h264 @ 0x563e577d8600] no frame! [in#0/rtsp @ 0x563e57796e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563e57796e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x563e5779b840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T03:28:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 199.589, "last_started_at": "2026-07-02T03:28:46+09:00", "pid": 35547, "process": {"exists": false, "pid": 35547}, "restart_count": 57, "restart_count_per_hour": 11, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 58}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.924, "sample_count": 2213, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 426506, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:32:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-02T03:32:06+09:00", "last_frame_read_age_sec": 0.062, "last_frame_read_at": "2026-07-02T03:32:06+09:00", "mjpeg": {"age_sec": 0.06, "exists": true, "mtime": "2026-07-02T03:32:06.455844+09:00", "mtime_epoch": 1782930726.4558437, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97252}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:32:06+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1070.715, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.423, "pushed": 0.423, "read": 0.458}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.485, "count": 12, "max_ms": 3.809, "p95_ms": 3.809}, "gstreamer_push": {"avg_ms": 1.75, "count": 12, "max_ms": 3.721, "p95_ms": 3.721}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.293, "count": 12, "max_ms": 72.553, "p95_ms": 72.553}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.88, "count": 12, "max_ms": 76.429, "p95_ms": 76.429}, "yolo11_640_fire_11class": {"avg_ms": 70.836, "count": 12, "max_ms": 77.842, "p95_ms": 77.842}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.984, "count": 12, "max_ms": 84.953, "p95_ms": 84.953}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.986, "count": 12, "max_ms": 97.662, "p95_ms": 97.662}, "read_interval": {"avg_ms": 2691.783, "count": 13, "max_ms": 27131.795, "p95_ms": 27131.795}, "triton_wait": {"avg_ms": 78.353, "count": 12, "max_ms": 84.966, "p95_ms": 84.966}, "visualize": {"avg_ms": 3.778, "count": 12, "max_ms": 3.988, "p95_ms": 3.988}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 28.385, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3554}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4264.539, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T03:32:06+09:00", "last_error_summary": "[NULL @ 0x563e5779b200] missing picture in access unit with size 18 [h264 @ 0x563e577d8600] no frame! [in#0/rtsp @ 0x563e57796e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563e57796e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x563e5779b840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 58, "h264_decode_error_count": 59, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:32:06+09:00", "last_error_summary": "[NULL @ 0x563e5779b200] missing picture in access unit with size 18 [h264 @ 0x563e577d8600] no frame! [in#0/rtsp @ 0x563e57796e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563e57796e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x563e5779b840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T03:32:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 199.63, "last_started_at": "2026-07-02T03:28:46+09:00", "pid": 35547, "process": {"exists": false, "pid": 35547}, "restart_count": 57, "restart_count_per_hour": 11, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 58}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.924, "sample_count": 2214, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 426507, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:32:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T03:32:06+09:00", "last_frame_read_age_sec": 0.103, "last_frame_read_at": "2026-07-02T03:32:06+09:00", "mjpeg": {"age_sec": 0.101, "exists": true, "mtime": "2026-07-02T03:32:06.455844+09:00", "mtime_epoch": 1782930726.4558437, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97252}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:32:06+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1070.715, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.457, "pushed": 0.457, "read": 0.457}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.495, "count": 13, "max_ms": 3.809, "p95_ms": 3.809}, "gstreamer_push": {"avg_ms": 1.721, "count": 13, "max_ms": 3.721, "p95_ms": 3.721}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.381, "count": 13, "max_ms": 73.44, "p95_ms": 73.44}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.086, "count": 13, "max_ms": 76.429, "p95_ms": 76.429}, "yolo11_640_fire_11class": {"avg_ms": 71.081, "count": 13, "max_ms": 77.842, "p95_ms": 77.842}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.352, "count": 13, "max_ms": 84.953, "p95_ms": 84.953}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.324, "count": 13, "max_ms": 97.662, "p95_ms": 97.662}, "read_interval": {"avg_ms": 2691.783, "count": 13, "max_ms": 27131.795, "p95_ms": 27131.795}, "triton_wait": {"avg_ms": 78.693, "count": 13, "max_ms": 84.966, "p95_ms": 84.966}, "visualize": {"avg_ms": 3.775, "count": 13, "max_ms": 3.988, "p95_ms": 3.988}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 28.425, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3554}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4264.5, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 58, "h264_decode_error_count": 59, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:32:06+09:00", "last_error_summary": "[NULL @ 0x563e5779b200] missing picture in access unit with size 18 [h264 @ 0x563e577d8600] no frame! [in#0/rtsp @ 0x563e57796e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563e57796e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x563e5779b840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T03:32:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 199.637, "last_started_at": "2026-07-02T03:28:46+09:00", "pid": 35547, "process": {"exists": false, "pid": 35547}, "restart_count": 57, "restart_count_per_hour": 11, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 58}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.924, "sample_count": 2214, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 426507, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:32:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-02T03:32:06+09:00", "last_frame_read_age_sec": 0.11, "last_frame_read_at": "2026-07-02T03:32:06+09:00", "mjpeg": {"age_sec": 0.108, "exists": true, "mtime": "2026-07-02T03:32:06.455844+09:00", "mtime_epoch": 1782930726.4558437, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97252}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:32:06+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1070.715, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.457, "pushed": 0.457, "read": 0.457}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.495, "count": 13, "max_ms": 3.809, "p95_ms": 3.809}, "gstreamer_push": {"avg_ms": 1.721, "count": 13, "max_ms": 3.721, "p95_ms": 3.721}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.381, "count": 13, "max_ms": 73.44, "p95_ms": 73.44}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.086, "count": 13, "max_ms": 76.429, "p95_ms": 76.429}, "yolo11_640_fire_11class": {"avg_ms": 71.081, "count": 13, "max_ms": 77.842, "p95_ms": 77.842}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.352, "count": 13, "max_ms": 84.953, "p95_ms": 84.953}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.324, "count": 13, "max_ms": 97.662, "p95_ms": 97.662}, "read_interval": {"avg_ms": 2691.783, "count": 13, "max_ms": 27131.795, "p95_ms": 27131.795}, "triton_wait": {"avg_ms": 78.693, "count": 13, "max_ms": 84.966, "p95_ms": 84.966}, "visualize": {"avg_ms": 3.775, "count": 13, "max_ms": 3.988, "p95_ms": 3.988}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 28.432, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3554}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4387.672, "limit_mb": 8796093022207.996}, "detail": {"pid": 35806, "restart_count": 58, "start_count": 59}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 58, "h264_decode_error_count": 59, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:32:06+09:00", "last_error_summary": "[NULL @ 0x563e5779b200] missing picture in access unit with size 18 [h264 @ 0x563e577d8600] no frame! [in#0/rtsp @ 0x563e57796e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563e57796e00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x563e5779b840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T03:32:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T03:32:09+09:00", "pid": 35806, "process": {"exists": true, "fd_count": 5, "hwm_mb": 156.207, "name": "ffmpeg", "pid": 35806, "rss_mb": 156.043, "state": "R (running)", "threads": 49}, "restart_count": 58, "restart_count_per_hour": 12, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": true, "start_count": 59}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.94, "sample_count": 2173, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 426507, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:32:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.479, "last_frame_pushed_at": "2026-07-02T03:32:06+09:00", "last_frame_read_age_sec": 4.577, "last_frame_read_at": "2026-07-02T03:32:06+09:00", "mjpeg": {"age_sec": 4.463, "exists": true, "mtime": "2026-07-02T03:32:06.567848+09:00", "mtime_epoch": 1782930726.567848, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97846}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:32:11+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1070.715, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 12, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.081, "pushed": 2.081, "read": 2.081}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.502, "count": 12, "max_ms": 3.809, "p95_ms": 3.809}, "gstreamer_push": {"avg_ms": 1.758, "count": 12, "max_ms": 3.721, "p95_ms": 3.721}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.626, "count": 12, "max_ms": 73.44, "p95_ms": 73.44}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.401, "count": 12, "max_ms": 76.429, "p95_ms": 76.429}, "yolo11_640_fire_11class": {"avg_ms": 72.312, "count": 12, "max_ms": 77.842, "p95_ms": 77.842}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.476, "count": 12, "max_ms": 84.953, "p95_ms": 84.953}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.211, "count": 12, "max_ms": 97.662, "p95_ms": 97.662}, "read_interval": {"avg_ms": 2360.153, "count": 12, "max_ms": 27131.795, "p95_ms": 27131.795}, "triton_wait": {"avg_ms": 80.557, "count": 12, "max_ms": 84.966, "p95_ms": 84.966}, "visualize": {"avg_ms": 3.777, "count": 12, "max_ms": 3.988, "p95_ms": 3.988}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 5.767, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3594}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4292.078, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 0, "rtsp_timeout_count": 2, "sample_message": "[in#0/rtsp @ 0x55d5c5e08e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d5c5e08e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 59, "h264_decode_error_count": 59, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:41:27+09:00", "last_error_summary": "[in#0/rtsp @ 0x55d5c5e08e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d5c5e08e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:32:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 558.331, "last_started_at": "2026-07-02T03:32:09+09:00", "pid": 35806, "process": {"exists": false, "pid": 35806}, "restart_count": 58, "restart_count_per_hour": 11, "rtsp_timeout_count": 106, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 59}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.115, "sample_count": 2512, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 431337, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:41:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.09, "last_frame_pushed_at": "2026-07-02T03:41:27+09:00", "last_frame_read_age_sec": 0.155, "last_frame_read_at": "2026-07-02T03:41:27+09:00", "mjpeg": {"age_sec": 0.081, "exists": true, "mtime": "2026-07-02T03:41:27.285834+09:00", "mtime_epoch": 1782931287.2858343, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 99173}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:41:27+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1073.141, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 1, "processed_frames": 368, "pushed_frames": 368, "read_frames": 368, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 12.271, "pushed": 12.271, "read": 12.271}, "hints": ["invalid_ffmpeg_reads_seen"], "latency_ms": {"analyze": {"avg_ms": 3.482, "count": 368, "max_ms": 5.365, "p95_ms": 3.644}, "gstreamer_push": {"avg_ms": 1.582, "count": 368, "max_ms": 6.067, "p95_ms": 3.617}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.198, "count": 368, "max_ms": 60.784, "p95_ms": 53.392}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.439, "count": 368, "max_ms": 61.899, "p95_ms": 57.808}, "yolo11_640_fire_11class": {"avg_ms": 51.391, "count": 368, "max_ms": 58.825, "p95_ms": 56.412}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.286, "count": 368, "max_ms": 60.124, "p95_ms": 57.264}}, "truncated_count": 0}, "processing_total": {"avg_ms": 65.276, "count": 368, "max_ms": 74.242, "p95_ms": 70.444}, "read_interval": {"avg_ms": 81.284, "count": 368, "max_ms": 92.438, "p95_ms": 87.318}, "triton_wait": {"avg_ms": 52.867, "count": 368, "max_ms": 61.912, "p95_ms": 57.933}, "visualize": {"avg_ms": 3.765, "count": 368, "max_ms": 4.725, "p95_ms": 3.994}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.99, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3594}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4291.891, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 59, "h264_decode_error_count": 59, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:41:27+09:00", "last_error_summary": "[in#0/rtsp @ 0x55d5c5e08e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d5c5e08e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:32:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 558.353, "last_started_at": "2026-07-02T03:32:09+09:00", "pid": null, "process": {"exists": false, "pid": 0}, "restart_count": 58, "restart_count_per_hour": 11, "rtsp_timeout_count": 106, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 59}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.115, "sample_count": 2511, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 431337, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:41:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.112, "last_frame_pushed_at": "2026-07-02T03:41:27+09:00", "last_frame_read_age_sec": 0.177, "last_frame_read_at": "2026-07-02T03:41:27+09:00", "mjpeg": {"age_sec": 0.097, "exists": true, "mtime": "2026-07-02T03:41:27.285834+09:00", "mtime_epoch": 1782931287.2858343, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 99173}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:41:27+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1073.141, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 1, "processed_frames": 368, "pushed_frames": 368, "read_frames": 367, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 12.291, "pushed": 12.291, "read": 12.257}, "hints": ["invalid_ffmpeg_reads_seen"], "latency_ms": {"analyze": {"avg_ms": 3.482, "count": 368, "max_ms": 5.365, "p95_ms": 3.644}, "gstreamer_push": {"avg_ms": 1.582, "count": 368, "max_ms": 6.067, "p95_ms": 3.617}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.198, "count": 368, "max_ms": 60.784, "p95_ms": 53.392}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.439, "count": 368, "max_ms": 61.899, "p95_ms": 57.808}, "yolo11_640_fire_11class": {"avg_ms": 51.391, "count": 368, "max_ms": 58.825, "p95_ms": 56.412}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.286, "count": 368, "max_ms": 60.124, "p95_ms": 57.264}}, "truncated_count": 0}, "processing_total": {"avg_ms": 65.276, "count": 368, "max_ms": 74.242, "p95_ms": 70.444}, "read_interval": {"avg_ms": 81.278, "count": 367, "max_ms": 92.438, "p95_ms": 87.318}, "triton_wait": {"avg_ms": 52.867, "count": 368, "max_ms": 61.912, "p95_ms": 57.933}, "visualize": {"avg_ms": 3.765, "count": 368, "max_ms": 4.725, "p95_ms": 3.994}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.941, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3594}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4441.0, "limit_mb": 8796093022207.996}, "detail": {"pid": 35958, "restart_count": 59, "start_count": 60}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 59, "h264_decode_error_count": 59, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:41:27+09:00", "last_error_summary": "[in#0/rtsp @ 0x55d5c5e08e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55d5c5e08e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:32:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T03:41:30+09:00", "pid": 35958, "process": {"exists": true, "fd_count": 5, "hwm_mb": 181.543, "name": "ffmpeg", "pid": 35958, "rss_mb": 181.543, "state": "R (running)", "threads": 106}, "restart_count": 59, "restart_count_per_hour": 12, "rtsp_timeout_count": 106, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": true, "start_count": 60}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.191, "sample_count": 2469, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 431337, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:41:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.807, "last_frame_pushed_at": "2026-07-02T03:41:27+09:00", "last_frame_read_age_sec": 4.872, "last_frame_read_at": "2026-07-02T03:41:27+09:00", "mjpeg": {"age_sec": 4.792, "exists": true, "mtime": "2026-07-02T03:41:27.285834+09:00", "mtime_epoch": 1782931287.2858343, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 99173}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:41:32+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1073.141, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 1, "processed_frames": 310, "pushed_frames": 310, "read_frames": 310, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 10.334, "pushed": 10.334, "read": 10.334}, "hints": ["invalid_ffmpeg_reads_seen"], "latency_ms": {"analyze": {"avg_ms": 3.471, "count": 310, "max_ms": 5.365, "p95_ms": 3.632}, "gstreamer_push": {"avg_ms": 1.579, "count": 310, "max_ms": 6.067, "p95_ms": 3.617}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.024, "count": 310, "max_ms": 57.798, "p95_ms": 53.317}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.201, "count": 310, "max_ms": 61.416, "p95_ms": 57.325}, "yolo11_640_fire_11class": {"avg_ms": 51.218, "count": 310, "max_ms": 58.825, "p95_ms": 56.068}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.192, "count": 310, "max_ms": 60.124, "p95_ms": 57.197}}, "truncated_count": 0}, "processing_total": {"avg_ms": 65.035, "count": 310, "max_ms": 74.15, "p95_ms": 70.312}, "read_interval": {"avg_ms": 81.322, "count": 310, "max_ms": 92.438, "p95_ms": 87.387}, "triton_wait": {"avg_ms": 52.647, "count": 310, "max_ms": 61.429, "p95_ms": 57.689}, "visualize": {"avg_ms": 3.76, "count": 310, "max_ms": 4.725, "p95_ms": 3.98}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3603}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4299.855, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55907ea045c0] error while decoding MB 55 54, bytestream -7 [in#0/rtsp @ 0x55907e889e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55907e889e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 60, "h264_decode_error_count": 60, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:44:08+09:00", "last_error_summary": "[h264 @ 0x55907ea045c0] error while decoding MB 55 54, bytestream -7 [in#0/rtsp @ 0x55907e889e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55907e889e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:32:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 158.223, "last_started_at": "2026-07-02T03:41:30+09:00", "pid": 35958, "process": {"exists": false, "pid": 35958}, "restart_count": 59, "restart_count_per_hour": 12, "rtsp_timeout_count": 108, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 60}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.538, "sample_count": 2151, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 432436, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:44:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.076, "last_frame_pushed_at": "2026-07-02T03:44:08+09:00", "last_frame_read_age_sec": 0.063, "last_frame_read_at": "2026-07-02T03:44:08+09:00", "mjpeg": {"age_sec": 0.058, "exists": true, "mtime": "2026-07-02T03:44:08.242134+09:00", "mtime_epoch": 1782931448.242134, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 107514}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:44:08+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1070.816, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.693, "pushed": 0.693, "read": 0.746}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.367, "count": 13, "max_ms": 3.438, "p95_ms": 3.438}, "gstreamer_push": {"avg_ms": 1.684, "count": 13, "max_ms": 6.09, "p95_ms": 6.09}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.09, "count": 13, "max_ms": 74.77, "p95_ms": 74.77}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.481, "count": 13, "max_ms": 82.208, "p95_ms": 82.208}, "yolo11_640_fire_11class": {"avg_ms": 69.125, "count": 13, "max_ms": 75.542, "p95_ms": 75.542}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.867, "count": 13, "max_ms": 86.607, "p95_ms": 86.607}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.919, "count": 13, "max_ms": 98.516, "p95_ms": 98.516}, "read_interval": {"avg_ms": 2675.751, "count": 14, "max_ms": 18755.679, "p95_ms": 18755.679}, "triton_wait": {"avg_ms": 78.636, "count": 13, "max_ms": 86.619, "p95_ms": 86.619}, "visualize": {"avg_ms": 3.732, "count": 13, "max_ms": 4.617, "p95_ms": 4.617}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.769, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3603}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4299.289, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T03:44:08+09:00", "last_error_summary": "[h264 @ 0x55907ea045c0] error while decoding MB 55 54, bytestream -7 [in#0/rtsp @ 0x55907e889e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55907e889e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 60, "h264_decode_error_count": 60, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:44:08+09:00", "last_error_summary": "[h264 @ 0x55907ea045c0] error while decoding MB 55 54, bytestream -7 [in#0/rtsp @ 0x55907e889e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55907e889e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:44:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 158.261, "last_started_at": "2026-07-02T03:41:30+09:00", "pid": 35958, "process": {"exists": false, "pid": 35958}, "restart_count": 59, "restart_count_per_hour": 12, "rtsp_timeout_count": 108, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 60}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.538, "sample_count": 2151, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 432437, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:44:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T03:44:08+09:00", "last_frame_read_age_sec": 0.102, "last_frame_read_at": "2026-07-02T03:44:08+09:00", "mjpeg": {"age_sec": 0.096, "exists": true, "mtime": "2026-07-02T03:44:08.242134+09:00", "mtime_epoch": 1782931448.242134, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 107514}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:44:08+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1070.816, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.744, "pushed": 0.744, "read": 0.744}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.366, "count": 14, "max_ms": 3.438, "p95_ms": 3.438}, "gstreamer_push": {"avg_ms": 1.663, "count": 14, "max_ms": 6.09, "p95_ms": 6.09}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.004, "count": 14, "max_ms": 74.77, "p95_ms": 74.77}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.603, "count": 14, "max_ms": 82.208, "p95_ms": 82.208}, "yolo11_640_fire_11class": {"avg_ms": 69.412, "count": 14, "max_ms": 75.542, "p95_ms": 75.542}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.136, "count": 14, "max_ms": 86.607, "p95_ms": 86.607}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.142, "count": 14, "max_ms": 98.516, "p95_ms": 98.516}, "read_interval": {"avg_ms": 2675.751, "count": 14, "max_ms": 18755.679, "p95_ms": 18755.679}, "triton_wait": {"avg_ms": 78.851, "count": 14, "max_ms": 86.619, "p95_ms": 86.619}, "visualize": {"avg_ms": 3.726, "count": 14, "max_ms": 4.617, "p95_ms": 4.617}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.807, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3603}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4299.039, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 60, "h264_decode_error_count": 60, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:44:08+09:00", "last_error_summary": "[h264 @ 0x55907ea045c0] error while decoding MB 55 54, bytestream -7 [in#0/rtsp @ 0x55907e889e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55907e889e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:44:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 158.269, "last_started_at": "2026-07-02T03:41:30+09:00", "pid": 35958, "process": {"exists": false, "pid": 35958}, "restart_count": 59, "restart_count_per_hour": 12, "rtsp_timeout_count": 108, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 60}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.538, "sample_count": 2151, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 432437, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:44:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-02T03:44:08+09:00", "last_frame_read_age_sec": 0.109, "last_frame_read_at": "2026-07-02T03:44:08+09:00", "mjpeg": {"age_sec": 0.103, "exists": true, "mtime": "2026-07-02T03:44:08.242134+09:00", "mtime_epoch": 1782931448.242134, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 107514}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:44:08+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1070.816, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.744, "pushed": 0.744, "read": 0.744}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.366, "count": 14, "max_ms": 3.438, "p95_ms": 3.438}, "gstreamer_push": {"avg_ms": 1.663, "count": 14, "max_ms": 6.09, "p95_ms": 6.09}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.004, "count": 14, "max_ms": 74.77, "p95_ms": 74.77}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.603, "count": 14, "max_ms": 82.208, "p95_ms": 82.208}, "yolo11_640_fire_11class": {"avg_ms": 69.412, "count": 14, "max_ms": 75.542, "p95_ms": 75.542}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.136, "count": 14, "max_ms": 86.607, "p95_ms": 86.607}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.142, "count": 14, "max_ms": 98.516, "p95_ms": 98.516}, "read_interval": {"avg_ms": 2675.751, "count": 14, "max_ms": 18755.679, "p95_ms": 18755.679}, "triton_wait": {"avg_ms": 78.851, "count": 14, "max_ms": 86.619, "p95_ms": 86.619}, "visualize": {"avg_ms": 3.726, "count": 14, "max_ms": 4.617, "p95_ms": 4.617}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.814, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3603}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4423.125, "limit_mb": 8796093022207.996}, "detail": {"pid": 36110, "restart_count": 60, "start_count": 61}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 60, "h264_decode_error_count": 60, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:44:08+09:00", "last_error_summary": "[h264 @ 0x55907ea045c0] error while decoding MB 55 54, bytestream -7 [in#0/rtsp @ 0x55907e889e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55907e889e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:44:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T03:44:11+09:00", "pid": 36110, "process": {"exists": true, "fd_count": 5, "hwm_mb": 157.027, "name": "ffmpeg", "pid": 36110, "rss_mb": 156.863, "state": "R (running)", "threads": 49}, "restart_count": 60, "restart_count_per_hour": 13, "rtsp_timeout_count": 108, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": true, "start_count": 61}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.589, "sample_count": 2109, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 432437, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:44:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.688, "last_frame_pushed_at": "2026-07-02T03:44:08+09:00", "last_frame_read_age_sec": 4.784, "last_frame_read_at": "2026-07-02T03:44:08+09:00", "mjpeg": {"age_sec": 4.671, "exists": true, "mtime": "2026-07-02T03:44:08.350137+09:00", "mtime_epoch": 1782931448.3501368, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 99424}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:44:13+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1070.816, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.596, "pushed": 0.596, "read": 0.596}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.366, "count": 14, "max_ms": 3.438, "p95_ms": 3.438}, "gstreamer_push": {"avg_ms": 1.663, "count": 14, "max_ms": 6.09, "p95_ms": 6.09}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.004, "count": 14, "max_ms": 74.77, "p95_ms": 74.77}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.603, "count": 14, "max_ms": 82.208, "p95_ms": 82.208}, "yolo11_640_fire_11class": {"avg_ms": 69.412, "count": 14, "max_ms": 75.542, "p95_ms": 75.542}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.136, "count": 14, "max_ms": 86.607, "p95_ms": 86.607}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.142, "count": 14, "max_ms": 98.516, "p95_ms": 98.516}, "read_interval": {"avg_ms": 2675.751, "count": 14, "max_ms": 18755.679, "p95_ms": 18755.679}, "triton_wait": {"avg_ms": 78.851, "count": 14, "max_ms": 86.619, "p95_ms": 86.619}, "visualize": {"avg_ms": 3.726, "count": 14, "max_ms": 4.617, "p95_ms": 4.617}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 23.489, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2636}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4323.48, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5644e29ca100] top block unavailable for requested intra mode -1 [h264 @ 0x5644e29ca100] error while decoding MB 51 0, bytestream 6304 [h264 @ 0x5644e282c400] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5644e282c400] error while decoding MB 0 9, bytestream 16411 [h264 @ 0x5644e28fb280] error while decoding MB 38 27, bytestream -5 [h264 @ 0x5644e29ca100] error while decoding MB 76 35, bytestream -20 [in#0/rtsp @ 0x5644e1c0be00] Error during demuxing: Connection timed ", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 203, "h264_decode_error_count": 369, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:46:00+09:00", "last_error_summary": "[h264 @ 0x5644e29ca100] top block unavailable for requested intra mode -1 [h264 @ 0x5644e29ca100] error while decoding MB 51 0, bytestream 6304 [h264 @ 0x5644e282c400] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5644e282c400] error while decoding MB 0 9, bytestream 16411 [h264 @ 0x5644e28fb280] error while decoding MB 38 27, bytestream -5 [h264 @ 0x5644e29ca100] error while decoding MB 76 35, bytestream -20 [in#0/rtsp @ 0x5644e1c0be00] Error during demuxing: Connection timed ", "last_exit_at": "2026-07-02T03:29:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 997.471, "last_started_at": "2026-07-02T03:29:23+09:00", "pid": 35699, "process": {"exists": false, "pid": 35699}, "restart_count": 228, "restart_count_per_hour": 4, "rtsp_timeout_count": 90, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 229}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.616, "sample_count": 2846, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 316438, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:46:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.082, "last_frame_pushed_at": "2026-07-02T03:46:00+09:00", "last_frame_read_age_sec": 0.068, "last_frame_read_at": "2026-07-02T03:46:00+09:00", "mjpeg": {"age_sec": 0.068, "exists": true, "mtime": "2026-07-02T03:46:00.477005+09:00", "mtime_epoch": 1782931560.4770052, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 125722}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T03:46:00+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1015.031, "name": "celery", "pid": 237, "rss_mb": 1010.25, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 105, "pushed_frames": 105, "read_frames": 105, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.503, "pushed": 3.503, "read": 3.503}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.503, "count": 105, "max_ms": 1.603, "p95_ms": 1.546}, "gstreamer_push": {"avg_ms": 1.511, "count": 105, "max_ms": 3.988, "p95_ms": 3.691}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.706, "count": 105, "max_ms": 84.477, "p95_ms": 76.306}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.186, "count": 105, "max_ms": 88.523, "p95_ms": 82.836}, "yolo11_640_fire_11class": {"avg_ms": 72.022, "count": 105, "max_ms": 84.49, "p95_ms": 76.968}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 81.364, "count": 105, "max_ms": 93.906, "p95_ms": 87.085}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.815, "count": 105, "max_ms": 103.994, "p95_ms": 97.926}, "read_interval": {"avg_ms": 285.63, "count": 105, "max_ms": 18717.618, "p95_ms": 115.375}, "triton_wait": {"avg_ms": 81.558, "count": 105, "max_ms": 93.917, "p95_ms": 87.129}, "visualize": {"avg_ms": 3.661, "count": 105, "max_ms": 4.182, "p95_ms": 3.796}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.972, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2636}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4323.031, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T03:46:00+09:00", "last_error_summary": "[h264 @ 0x5644e29ca100] top block unavailable for requested intra mode -1 [h264 @ 0x5644e29ca100] error while decoding MB 51 0, bytestream 6304 [h264 @ 0x5644e282c400] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5644e282c400] error while decoding MB 0 9, bytestream 16411 [h264 @ 0x5644e28fb280] error while decoding MB 38 27, bytestream -5 [h264 @ 0x5644e29ca100] error while decoding MB 76 35, bytestream -20 [in#0/rtsp @ 0x5644e1c0be00] Error during demuxing: Connection timed ", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 203, "h264_decode_error_count": 369, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:46:00+09:00", "last_error_summary": "[h264 @ 0x5644e29ca100] top block unavailable for requested intra mode -1 [h264 @ 0x5644e29ca100] error while decoding MB 51 0, bytestream 6304 [h264 @ 0x5644e282c400] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5644e282c400] error while decoding MB 0 9, bytestream 16411 [h264 @ 0x5644e28fb280] error while decoding MB 38 27, bytestream -5 [h264 @ 0x5644e29ca100] error while decoding MB 76 35, bytestream -20 [in#0/rtsp @ 0x5644e1c0be00] Error during demuxing: Connection timed ", "last_exit_at": "2026-07-02T03:46:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 997.506, "last_started_at": "2026-07-02T03:29:23+09:00", "pid": 35699, "process": {"exists": false, "pid": 35699}, "restart_count": 228, "restart_count_per_hour": 4, "rtsp_timeout_count": 90, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 229}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.616, "sample_count": 2846, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 316439, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:46:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T03:46:00+09:00", "last_frame_read_age_sec": 0.104, "last_frame_read_at": "2026-07-02T03:46:00+09:00", "mjpeg": {"age_sec": 0.104, "exists": true, "mtime": "2026-07-02T03:46:00.477005+09:00", "mtime_epoch": 1782931560.4770052, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 125722}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T03:46:00+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1015.031, "name": "celery", "pid": 237, "rss_mb": 1010.25, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 105, "pushed_frames": 105, "read_frames": 105, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.501, "pushed": 3.501, "read": 3.501}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.502, "count": 105, "max_ms": 1.603, "p95_ms": 1.546}, "gstreamer_push": {"avg_ms": 1.486, "count": 105, "max_ms": 3.988, "p95_ms": 1.609}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.599, "count": 105, "max_ms": 84.477, "p95_ms": 76.306}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.206, "count": 105, "max_ms": 88.523, "p95_ms": 82.836}, "yolo11_640_fire_11class": {"avg_ms": 72.114, "count": 105, "max_ms": 84.49, "p95_ms": 76.968}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 81.44, "count": 105, "max_ms": 93.906, "p95_ms": 87.085}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.884, "count": 105, "max_ms": 103.994, "p95_ms": 97.926}, "read_interval": {"avg_ms": 285.63, "count": 105, "max_ms": 18717.618, "p95_ms": 115.375}, "triton_wait": {"avg_ms": 81.628, "count": 105, "max_ms": 93.917, "p95_ms": 87.129}, "visualize": {"avg_ms": 3.66, "count": 105, "max_ms": 4.182, "p95_ms": 3.796}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.989, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2636}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4323.07, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 203, "h264_decode_error_count": 369, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:46:00+09:00", "last_error_summary": "[h264 @ 0x5644e29ca100] top block unavailable for requested intra mode -1 [h264 @ 0x5644e29ca100] error while decoding MB 51 0, bytestream 6304 [h264 @ 0x5644e282c400] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5644e282c400] error while decoding MB 0 9, bytestream 16411 [h264 @ 0x5644e28fb280] error while decoding MB 38 27, bytestream -5 [h264 @ 0x5644e29ca100] error while decoding MB 76 35, bytestream -20 [in#0/rtsp @ 0x5644e1c0be00] Error during demuxing: Connection timed ", "last_exit_at": "2026-07-02T03:46:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 997.514, "last_started_at": "2026-07-02T03:29:23+09:00", "pid": 35699, "process": {"exists": false, "pid": 35699}, "restart_count": 228, "restart_count_per_hour": 4, "rtsp_timeout_count": 90, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 229}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.616, "sample_count": 2846, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 316439, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:46:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-02T03:46:00+09:00", "last_frame_read_age_sec": 0.112, "last_frame_read_at": "2026-07-02T03:46:00+09:00", "mjpeg": {"age_sec": 0.112, "exists": true, "mtime": "2026-07-02T03:46:00.477005+09:00", "mtime_epoch": 1782931560.4770052, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 125722}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T03:46:00+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1015.031, "name": "celery", "pid": 237, "rss_mb": 1010.25, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 105, "pushed_frames": 105, "read_frames": 105, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.5, "pushed": 3.5, "read": 3.5}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.502, "count": 105, "max_ms": 1.603, "p95_ms": 1.546}, "gstreamer_push": {"avg_ms": 1.486, "count": 105, "max_ms": 3.988, "p95_ms": 1.609}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.599, "count": 105, "max_ms": 84.477, "p95_ms": 76.306}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.206, "count": 105, "max_ms": 88.523, "p95_ms": 82.836}, "yolo11_640_fire_11class": {"avg_ms": 72.114, "count": 105, "max_ms": 84.49, "p95_ms": 76.968}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 81.44, "count": 105, "max_ms": 93.906, "p95_ms": 87.085}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.884, "count": 105, "max_ms": 103.994, "p95_ms": 97.926}, "read_interval": {"avg_ms": 285.63, "count": 105, "max_ms": 18717.618, "p95_ms": 115.375}, "triton_wait": {"avg_ms": 81.628, "count": 105, "max_ms": 93.917, "p95_ms": 87.129}, "visualize": {"avg_ms": 3.66, "count": 105, "max_ms": 4.182, "p95_ms": 3.796}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2636}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4427.906, "limit_mb": 8796093022207.996}, "detail": {"pid": 36262, "restart_count": 229, "start_count": 230}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 203, "h264_decode_error_count": 369, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:46:00+09:00", "last_error_summary": "[h264 @ 0x5644e29ca100] top block unavailable for requested intra mode -1 [h264 @ 0x5644e29ca100] error while decoding MB 51 0, bytestream 6304 [h264 @ 0x5644e282c400] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5644e282c400] error while decoding MB 0 9, bytestream 16411 [h264 @ 0x5644e28fb280] error while decoding MB 38 27, bytestream -5 [h264 @ 0x5644e29ca100] error while decoding MB 76 35, bytestream -20 [in#0/rtsp @ 0x5644e1c0be00] Error during demuxing: Connection timed ", "last_exit_at": "2026-07-02T03:46:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T03:46:02+09:00", "pid": 36262, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.801, "name": "ffmpeg", "pid": 36262, "rss_mb": 138.801, "state": "S (sleeping)", "threads": 106}, "restart_count": 229, "restart_count_per_hour": 5, "rtsp_timeout_count": 90, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 230}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.623, "sample_count": 2796, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 316439, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:46:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.179, "last_frame_pushed_at": "2026-07-02T03:46:00+09:00", "last_frame_read_age_sec": 4.277, "last_frame_read_at": "2026-07-02T03:46:00+09:00", "mjpeg": {"age_sec": 4.165, "exists": true, "mtime": "2026-07-02T03:46:00.589008+09:00", "mtime_epoch": 1782931560.589008, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 127410}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T03:46:04+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1015.031, "name": "celery", "pid": 237, "rss_mb": 1010.25, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 68, "pushed_frames": 68, "read_frames": 67, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.267, "pushed": 2.267, "read": 2.234}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.504, "count": 68, "max_ms": 1.603, "p95_ms": 1.543}, "gstreamer_push": {"avg_ms": 1.427, "count": 68, "max_ms": 3.946, "p95_ms": 1.471}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.863, "count": 68, "max_ms": 84.477, "p95_ms": 76.982}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.289, "count": 68, "max_ms": 88.523, "p95_ms": 82.836}, "yolo11_640_fire_11class": {"avg_ms": 71.576, "count": 68, "max_ms": 84.49, "p95_ms": 77.703}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.768, "count": 68, "max_ms": 93.906, "p95_ms": 87.688}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.177, "count": 68, "max_ms": 103.994, "p95_ms": 97.974}, "read_interval": {"avg_ms": 385.271, "count": 67, "max_ms": 18717.618, "p95_ms": 115.551}, "triton_wait": {"avg_ms": 80.964, "count": 68, "max_ms": 93.917, "p95_ms": 87.701}, "visualize": {"avg_ms": 3.647, "count": 68, "max_ms": 4.182, "p95_ms": 3.776}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2652}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4326.477, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5594ee53fb80] reference picture missing during reorder [h264 @ 0x5594ee53fb80] Missing reference picture, default is 65540 [h264 @ 0x5594ee553d40] mmco: unref short failure [h264 @ 0x5594ee553d40] illegal short term buffer state detected [h264 @ 0x5594ee426480] left block unavailable for requested intra mode [h264 @ 0x5594ee426480] error while decoding MB 0 37, bytestream 84973 [in#0/rtsp @ 0x5594ee40be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5594ee40be00] Error r", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 204, "h264_decode_error_count": 370, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:49:49+09:00", "last_error_summary": "[h264 @ 0x5594ee53fb80] reference picture missing during reorder [h264 @ 0x5594ee53fb80] Missing reference picture, default is 65540 [h264 @ 0x5594ee553d40] mmco: unref short failure [h264 @ 0x5594ee553d40] illegal short term buffer state detected [h264 @ 0x5594ee426480] left block unavailable for requested intra mode [h264 @ 0x5594ee426480] error while decoding MB 0 37, bytestream 84973 [in#0/rtsp @ 0x5594ee40be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5594ee40be00] Error r", "last_exit_at": "2026-07-02T03:46:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 226.864, "last_started_at": "2026-07-02T03:46:02+09:00", "pid": 36262, "process": {"exists": false, "pid": 36262}, "restart_count": 229, "restart_count_per_hour": 5, "rtsp_timeout_count": 92, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 230}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.334, "sample_count": 2346, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 318322, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:49:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.067, "last_frame_pushed_at": "2026-07-02T03:49:49+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-02T03:49:49+09:00", "mjpeg": {"age_sec": 0.05, "exists": true, "mtime": "2026-07-02T03:49:49.566632+09:00", "mtime_epoch": 1782931789.5666318, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 115804}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T03:49:49+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1015.805, "name": "celery", "pid": 237, "rss_mb": 1011.48, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 108, "pushed_frames": 108, "read_frames": 108, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.61, "pushed": 3.61, "read": 3.61}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.533, "count": 109, "max_ms": 2.131, "p95_ms": 1.625}, "gstreamer_push": {"avg_ms": 1.609, "count": 108, "max_ms": 6.226, "p95_ms": 3.81}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.172, "count": 109, "max_ms": 87.001, "p95_ms": 77.472}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.271, "count": 109, "max_ms": 87.013, "p95_ms": 81.384}, "yolo11_640_fire_11class": {"avg_ms": 64.822, "count": 109, "max_ms": 85.809, "p95_ms": 79.459}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.328, "count": 109, "max_ms": 94.589, "p95_ms": 86.422}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.905, "count": 108, "max_ms": 105.048, "p95_ms": 97.034}, "read_interval": {"avg_ms": 277.341, "count": 108, "max_ms": 19528.337, "p95_ms": 115.557}, "triton_wait": {"avg_ms": 70.452, "count": 109, "max_ms": 94.602, "p95_ms": 86.435}, "visualize": {"avg_ms": 3.633, "count": 109, "max_ms": 4.409, "p95_ms": 3.79}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.915, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2652}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4326.191, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T03:49:49+09:00", "last_error_summary": "[h264 @ 0x5594ee53fb80] reference picture missing during reorder [h264 @ 0x5594ee53fb80] Missing reference picture, default is 65540 [h264 @ 0x5594ee553d40] mmco: unref short failure [h264 @ 0x5594ee553d40] illegal short term buffer state detected [h264 @ 0x5594ee426480] left block unavailable for requested intra mode [h264 @ 0x5594ee426480] error while decoding MB 0 37, bytestream 84973 [in#0/rtsp @ 0x5594ee40be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5594ee40be00] Error r", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 204, "h264_decode_error_count": 370, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:49:49+09:00", "last_error_summary": "[h264 @ 0x5594ee53fb80] reference picture missing during reorder [h264 @ 0x5594ee53fb80] Missing reference picture, default is 65540 [h264 @ 0x5594ee553d40] mmco: unref short failure [h264 @ 0x5594ee553d40] illegal short term buffer state detected [h264 @ 0x5594ee426480] left block unavailable for requested intra mode [h264 @ 0x5594ee426480] error while decoding MB 0 37, bytestream 84973 [in#0/rtsp @ 0x5594ee40be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5594ee40be00] Error r", "last_exit_at": "2026-07-02T03:49:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 226.882, "last_started_at": "2026-07-02T03:46:02+09:00", "pid": 36262, "process": {"exists": false, "pid": 36262}, "restart_count": 229, "restart_count_per_hour": 5, "rtsp_timeout_count": 92, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 230}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.334, "sample_count": 2347, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 318322, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:49:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T03:49:49+09:00", "last_frame_read_age_sec": 0.072, "last_frame_read_at": "2026-07-02T03:49:49+09:00", "mjpeg": {"age_sec": 0.068, "exists": true, "mtime": "2026-07-02T03:49:49.566632+09:00", "mtime_epoch": 1782931789.5666318, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 115804}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T03:49:49+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1015.805, "name": "celery", "pid": 237, "rss_mb": 1011.484, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 109, "pushed_frames": 109, "read_frames": 108, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.642, "pushed": 3.642, "read": 3.608}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.533, "count": 109, "max_ms": 2.131, "p95_ms": 1.625}, "gstreamer_push": {"avg_ms": 1.607, "count": 109, "max_ms": 6.226, "p95_ms": 3.81}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.172, "count": 109, "max_ms": 87.001, "p95_ms": 77.472}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.271, "count": 109, "max_ms": 87.013, "p95_ms": 81.384}, "yolo11_640_fire_11class": {"avg_ms": 64.822, "count": 109, "max_ms": 85.809, "p95_ms": 79.459}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.328, "count": 109, "max_ms": 94.589, "p95_ms": 86.422}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.749, "count": 109, "max_ms": 105.048, "p95_ms": 97.034}, "read_interval": {"avg_ms": 277.341, "count": 108, "max_ms": 19528.337, "p95_ms": 115.557}, "triton_wait": {"avg_ms": 70.452, "count": 109, "max_ms": 94.602, "p95_ms": 86.435}, "visualize": {"avg_ms": 3.633, "count": 109, "max_ms": 4.409, "p95_ms": 3.79}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.932, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2652}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4326.152, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 204, "h264_decode_error_count": 370, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:49:49+09:00", "last_error_summary": "[h264 @ 0x5594ee53fb80] reference picture missing during reorder [h264 @ 0x5594ee53fb80] Missing reference picture, default is 65540 [h264 @ 0x5594ee553d40] mmco: unref short failure [h264 @ 0x5594ee553d40] illegal short term buffer state detected [h264 @ 0x5594ee426480] left block unavailable for requested intra mode [h264 @ 0x5594ee426480] error while decoding MB 0 37, bytestream 84973 [in#0/rtsp @ 0x5594ee40be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5594ee40be00] Error r", "last_exit_at": "2026-07-02T03:49:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 226.889, "last_started_at": "2026-07-02T03:46:02+09:00", "pid": 36262, "process": {"exists": false, "pid": 36262}, "restart_count": 229, "restart_count_per_hour": 5, "rtsp_timeout_count": 92, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 230}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.334, "sample_count": 2347, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 318322, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:49:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-02T03:49:49+09:00", "last_frame_read_age_sec": 0.08, "last_frame_read_at": "2026-07-02T03:49:49+09:00", "mjpeg": {"age_sec": 0.075, "exists": true, "mtime": "2026-07-02T03:49:49.566632+09:00", "mtime_epoch": 1782931789.5666318, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 115804}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T03:49:49+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1015.805, "name": "celery", "pid": 237, "rss_mb": 1011.484, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 109, "pushed_frames": 109, "read_frames": 108, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.641, "pushed": 3.641, "read": 3.607}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.533, "count": 109, "max_ms": 2.131, "p95_ms": 1.625}, "gstreamer_push": {"avg_ms": 1.607, "count": 109, "max_ms": 6.226, "p95_ms": 3.81}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.172, "count": 109, "max_ms": 87.001, "p95_ms": 77.472}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.271, "count": 109, "max_ms": 87.013, "p95_ms": 81.384}, "yolo11_640_fire_11class": {"avg_ms": 64.822, "count": 109, "max_ms": 85.809, "p95_ms": 79.459}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.328, "count": 109, "max_ms": 94.589, "p95_ms": 86.422}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.749, "count": 109, "max_ms": 105.048, "p95_ms": 97.034}, "read_interval": {"avg_ms": 277.341, "count": 108, "max_ms": 19528.337, "p95_ms": 115.557}, "triton_wait": {"avg_ms": 70.452, "count": 109, "max_ms": 94.602, "p95_ms": 86.435}, "visualize": {"avg_ms": 3.633, "count": 109, "max_ms": 4.409, "p95_ms": 3.79}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.94, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3628}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4020.52, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55a4d339b200] error while decoding MB 95 36, bytestream -9 [in#0/rtsp @ 0x55a4d335be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a4d335be00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 61, "h264_decode_error_count": 61, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:49:51+09:00", "last_error_summary": "[h264 @ 0x55a4d339b200] error while decoding MB 95 36, bytestream -9 [in#0/rtsp @ 0x55a4d335be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a4d335be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:44:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 340.449, "last_started_at": "2026-07-02T03:44:11+09:00", "pid": 36110, "process": {"exists": false, "pid": 36110}, "restart_count": 60, "restart_count_per_hour": 12, "rtsp_timeout_count": 110, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 61}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.095, "sample_count": 2598, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 435392, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:49:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T03:49:51+09:00", "last_frame_read_age_sec": 0.069, "last_frame_read_at": "2026-07-02T03:49:51+09:00", "mjpeg": {"age_sec": 0.067, "exists": true, "mtime": "2026-07-02T03:49:51.402676+09:00", "mtime_epoch": 1782931791.4026759, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 104394}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:49:51+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1072.012, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 65, "pushed_frames": 65, "read_frames": 64, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.167, "pushed": 2.167, "read": 2.134}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.409, "count": 65, "max_ms": 4.685, "p95_ms": 3.461}, "gstreamer_push": {"avg_ms": 1.7, "count": 65, "max_ms": 6.348, "p95_ms": 3.887}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.979, "count": 65, "max_ms": 84.423, "p95_ms": 75.953}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.337, "count": 65, "max_ms": 84.919, "p95_ms": 78.662}, "yolo11_640_fire_11class": {"avg_ms": 68.292, "count": 65, "max_ms": 83.792, "p95_ms": 78.607}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.644, "count": 65, "max_ms": 92.587, "p95_ms": 85.47}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.174, "count": 65, "max_ms": 104.787, "p95_ms": 97.962}, "read_interval": {"avg_ms": 469.007, "count": 64, "max_ms": 14516.891, "p95_ms": 121.557}, "triton_wait": {"avg_ms": 74.903, "count": 65, "max_ms": 92.599, "p95_ms": 85.482}, "visualize": {"avg_ms": 3.659, "count": 65, "max_ms": 4.66, "p95_ms": 3.836}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3628}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4020.5, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T03:49:51+09:00", "last_error_summary": "[h264 @ 0x55a4d339b200] error while decoding MB 95 36, bytestream -9 [in#0/rtsp @ 0x55a4d335be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a4d335be00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 61, "h264_decode_error_count": 61, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:49:51+09:00", "last_error_summary": "[h264 @ 0x55a4d339b200] error while decoding MB 95 36, bytestream -9 [in#0/rtsp @ 0x55a4d335be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a4d335be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:49:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 340.459, "last_started_at": "2026-07-02T03:44:11+09:00", "pid": 36110, "process": {"exists": false, "pid": 36110}, "restart_count": 60, "restart_count_per_hour": 12, "rtsp_timeout_count": 110, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 61}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.095, "sample_count": 2598, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 435392, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:49:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-02T03:49:51+09:00", "last_frame_read_age_sec": 0.078, "last_frame_read_at": "2026-07-02T03:49:51+09:00", "mjpeg": {"age_sec": 0.076, "exists": true, "mtime": "2026-07-02T03:49:51.402676+09:00", "mtime_epoch": 1782931791.4026759, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 104394}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:49:51+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1072.012, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 65, "read_frames": 64, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.133, "pushed": 2.167, "read": 2.133}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.411, "count": 64, "max_ms": 4.685, "p95_ms": 3.461}, "gstreamer_push": {"avg_ms": 1.7, "count": 65, "max_ms": 6.348, "p95_ms": 3.887}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.85, "count": 64, "max_ms": 84.423, "p95_ms": 75.953}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.239, "count": 64, "max_ms": 84.919, "p95_ms": 78.662}, "yolo11_640_fire_11class": {"avg_ms": 68.255, "count": 64, "max_ms": 83.792, "p95_ms": 78.607}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.547, "count": 64, "max_ms": 92.587, "p95_ms": 85.47}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.08, "count": 64, "max_ms": 104.787, "p95_ms": 97.962}, "read_interval": {"avg_ms": 469.007, "count": 64, "max_ms": 14516.891, "p95_ms": 121.557}, "triton_wait": {"avg_ms": 74.81, "count": 64, "max_ms": 92.599, "p95_ms": 85.482}, "visualize": {"avg_ms": 3.658, "count": 64, "max_ms": 4.66, "p95_ms": 3.836}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3628}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4020.469, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 61, "h264_decode_error_count": 61, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:49:51+09:00", "last_error_summary": "[h264 @ 0x55a4d339b200] error while decoding MB 95 36, bytestream -9 [in#0/rtsp @ 0x55a4d335be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a4d335be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:49:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 340.467, "last_started_at": "2026-07-02T03:44:11+09:00", "pid": 36110, "process": {"exists": false, "pid": 36110}, "restart_count": 60, "restart_count_per_hour": 12, "rtsp_timeout_count": 110, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": false, "start_count": 61}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.095, "sample_count": 2597, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 435392, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:49:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.024, "last_frame_pushed_at": "2026-07-02T03:49:51+09:00", "last_frame_read_age_sec": 0.086, "last_frame_read_at": "2026-07-02T03:49:51+09:00", "mjpeg": {"age_sec": 0.008, "exists": true, "mtime": "2026-07-02T03:49:51.478678+09:00", "mtime_epoch": 1782931791.4786777, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 105187}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:49:51+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1072.012, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 64, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.134, "pushed": 2.134, "read": 2.134}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.411, "count": 64, "max_ms": 4.685, "p95_ms": 3.461}, "gstreamer_push": {"avg_ms": 1.706, "count": 64, "max_ms": 6.348, "p95_ms": 3.887}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.85, "count": 64, "max_ms": 84.423, "p95_ms": 75.953}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.239, "count": 64, "max_ms": 84.919, "p95_ms": 78.662}, "yolo11_640_fire_11class": {"avg_ms": 68.255, "count": 64, "max_ms": 83.792, "p95_ms": 78.607}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.547, "count": 64, "max_ms": 92.587, "p95_ms": 85.47}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.08, "count": 64, "max_ms": 104.787, "p95_ms": 97.962}, "read_interval": {"avg_ms": 469.007, "count": 64, "max_ms": 14516.891, "p95_ms": 121.557}, "triton_wait": {"avg_ms": 74.81, "count": 64, "max_ms": 92.599, "p95_ms": 85.482}, "visualize": {"avg_ms": 3.658, "count": 64, "max_ms": 4.66, "p95_ms": 3.836}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.99, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2652}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4128.902, "limit_mb": 8796093022207.996}, "detail": {"pid": 36369, "restart_count": 230, "start_count": 231}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 204, "h264_decode_error_count": 370, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:49:49+09:00", "last_error_summary": "[h264 @ 0x5594ee53fb80] reference picture missing during reorder [h264 @ 0x5594ee53fb80] Missing reference picture, default is 65540 [h264 @ 0x5594ee553d40] mmco: unref short failure [h264 @ 0x5594ee553d40] illegal short term buffer state detected [h264 @ 0x5594ee426480] left block unavailable for requested intra mode [h264 @ 0x5594ee426480] error while decoding MB 0 37, bytestream 84973 [in#0/rtsp @ 0x5594ee40be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5594ee40be00] Error r", "last_exit_at": "2026-07-02T03:49:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T03:49:51+09:00", "pid": 36369, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.312, "name": "ffmpeg", "pid": 36369, "rss_mb": 139.312, "state": "S (sleeping)", "threads": 106}, "restart_count": 230, "restart_count_per_hour": 6, "rtsp_timeout_count": 92, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 231}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.373, "sample_count": 2310, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 318322, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:49:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.171, "last_frame_pushed_at": "2026-07-02T03:49:49+09:00", "last_frame_read_age_sec": 4.237, "last_frame_read_at": "2026-07-02T03:49:49+09:00", "mjpeg": {"age_sec": 4.157, "exists": true, "mtime": "2026-07-02T03:49:49.642634+09:00", "mtime_epoch": 1782931789.6426337, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 115107}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T03:49:53+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1015.805, "name": "celery", "pid": 237, "rss_mb": 1011.484, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 72, "pushed_frames": 72, "read_frames": 71, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.4, "pushed": 2.4, "read": 2.367}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.516, "count": 72, "max_ms": 1.869, "p95_ms": 1.579}, "gstreamer_push": {"avg_ms": 1.549, "count": 72, "max_ms": 5.68, "p95_ms": 3.682}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.788, "count": 72, "max_ms": 87.001, "p95_ms": 77.661}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.262, "count": 72, "max_ms": 87.013, "p95_ms": 78.64}, "yolo11_640_fire_11class": {"avg_ms": 59.634, "count": 72, "max_ms": 85.809, "p95_ms": 78.651}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.539, "count": 72, "max_ms": 94.589, "p95_ms": 86.422}}, "truncated_count": 0}, "processing_total": {"avg_ms": 73.884, "count": 72, "max_ms": 105.048, "p95_ms": 96.997}, "read_interval": {"avg_ms": 364.119, "count": 71, "max_ms": 19528.337, "p95_ms": 114.987}, "triton_wait": {"avg_ms": 63.681, "count": 72, "max_ms": 94.602, "p95_ms": 86.435}, "visualize": {"avg_ms": 3.597, "count": 72, "max_ms": 4.25, "p95_ms": 3.758}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3628}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4198.703, "limit_mb": 8796093022207.996}, "detail": {"pid": 36476, "restart_count": 61, "start_count": 62}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 61, "h264_decode_error_count": 61, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:49:51+09:00", "last_error_summary": "[h264 @ 0x55a4d339b200] error while decoding MB 95 36, bytestream -9 [in#0/rtsp @ 0x55a4d335be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a4d335be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:49:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T03:49:53+09:00", "pid": 36476, "process": {"exists": true, "fd_count": 5, "hwm_mb": 71.023, "name": "ffmpeg", "pid": 36476, "rss_mb": 71.023, "state": "S (sleeping)", "threads": 19}, "restart_count": 61, "restart_count_per_hour": 13, "rtsp_timeout_count": 110, "rtsp_timeout_count_per_hour": 24, "rtsp_tolerant_mode": false, "running": true, "start_count": 62}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.065, "sample_count": 2559, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 435392, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:49:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.34, "last_frame_pushed_at": "2026-07-02T03:49:51+09:00", "last_frame_read_age_sec": 4.402, "last_frame_read_at": "2026-07-02T03:49:51+09:00", "mjpeg": {"age_sec": 4.324, "exists": true, "mtime": "2026-07-02T03:49:51.478678+09:00", "mtime_epoch": 1782931791.4786777, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 105187}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:49:55+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1072.012, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 25, "pushed_frames": 26, "read_frames": 25, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.833, "pushed": 0.867, "read": 0.833}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.406, "count": 25, "max_ms": 3.466, "p95_ms": 3.458}, "gstreamer_push": {"avg_ms": 1.609, "count": 26, "max_ms": 3.963, "p95_ms": 3.88}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.104, "count": 25, "max_ms": 76.851, "p95_ms": 75.953}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.295, "count": 25, "max_ms": 78.596, "p95_ms": 76.669}, "yolo11_640_fire_11class": {"avg_ms": 59.57, "count": 25, "max_ms": 78.607, "p95_ms": 75.966}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.872, "count": 25, "max_ms": 85.906, "p95_ms": 84.497}}, "truncated_count": 0}, "processing_total": {"avg_ms": 76.563, "count": 25, "max_ms": 98.362, "p95_ms": 96.517}, "read_interval": {"avg_ms": 1027.576, "count": 25, "max_ms": 14516.891, "p95_ms": 9020.02}, "triton_wait": {"avg_ms": 64.389, "count": 25, "max_ms": 85.918, "p95_ms": 84.509}, "visualize": {"avg_ms": 3.687, "count": 25, "max_ms": 4.66, "p95_ms": 3.827}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3637}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4281.344, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x559e95d55180] error while decoding MB 5 65, bytestream -5 [in#0/rtsp @ 0x559e95ccde00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559e95ccde00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 62, "h264_decode_error_count": 62, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:52:29+09:00", "last_error_summary": "[h264 @ 0x559e95d55180] error while decoding MB 5 65, bytestream -5 [in#0/rtsp @ 0x559e95ccde00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559e95ccde00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:49:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 155.749, "last_started_at": "2026-07-02T03:49:53+09:00", "pid": 36476, "process": {"exists": false, "pid": 36476}, "restart_count": 61, "restart_count_per_hour": 13, "rtsp_timeout_count": 112, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": false, "start_count": 62}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.592, "sample_count": 2190, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 436501, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:52:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-02T03:52:29+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-02T03:52:29+09:00", "mjpeg": {"age_sec": 0.06, "exists": true, "mtime": "2026-07-02T03:52:29.490425+09:00", "mtime_epoch": 1782931949.4904249, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 104769}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:52:29+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1078.0, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.737, "pushed": 0.737, "read": 0.793}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.427, "count": 13, "max_ms": 3.524, "p95_ms": 3.524}, "gstreamer_push": {"avg_ms": 1.559, "count": 13, "max_ms": 3.959, "p95_ms": 3.959}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 65.098, "count": 13, "max_ms": 77.777, "p95_ms": 77.777}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.814, "count": 13, "max_ms": 81.946, "p95_ms": 81.946}, "yolo11_640_fire_11class": {"avg_ms": 70.132, "count": 13, "max_ms": 78.882, "p95_ms": 78.882}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.428, "count": 13, "max_ms": 89.753, "p95_ms": 89.753}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.156, "count": 13, "max_ms": 102.274, "p95_ms": 102.274}, "read_interval": {"avg_ms": 2376.487, "count": 14, "max_ms": 16301.165, "p95_ms": 16301.165}, "triton_wait": {"avg_ms": 78.638, "count": 13, "max_ms": 89.765, "p95_ms": 89.765}, "visualize": {"avg_ms": 3.821, "count": 13, "max_ms": 4.111, "p95_ms": 4.111}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.644, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3637}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4283.715, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T03:52:29+09:00", "last_error_summary": "[h264 @ 0x559e95d55180] error while decoding MB 5 65, bytestream -5 [in#0/rtsp @ 0x559e95ccde00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559e95ccde00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 62, "h264_decode_error_count": 62, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:52:29+09:00", "last_error_summary": "[h264 @ 0x559e95d55180] error while decoding MB 5 65, bytestream -5 [in#0/rtsp @ 0x559e95ccde00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559e95ccde00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:52:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 155.788, "last_started_at": "2026-07-02T03:49:53+09:00", "pid": 36476, "process": {"exists": false, "pid": 36476}, "restart_count": 61, "restart_count_per_hour": 13, "rtsp_timeout_count": 112, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": false, "start_count": 62}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.592, "sample_count": 2190, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 436502, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:52:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T03:52:29+09:00", "last_frame_read_age_sec": 0.104, "last_frame_read_at": "2026-07-02T03:52:29+09:00", "mjpeg": {"age_sec": 0.1, "exists": true, "mtime": "2026-07-02T03:52:29.490425+09:00", "mtime_epoch": 1782931949.4904249, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 104769}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:52:29+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1078.0, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.792, "pushed": 0.792, "read": 0.792}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.424, "count": 14, "max_ms": 3.524, "p95_ms": 3.524}, "gstreamer_push": {"avg_ms": 1.54, "count": 14, "max_ms": 3.959, "p95_ms": 3.959}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 65.304, "count": 14, "max_ms": 77.777, "p95_ms": 77.777}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.011, "count": 14, "max_ms": 81.946, "p95_ms": 81.946}, "yolo11_640_fire_11class": {"avg_ms": 70.428, "count": 14, "max_ms": 78.882, "p95_ms": 78.882}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.875, "count": 14, "max_ms": 89.753, "p95_ms": 89.753}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.573, "count": 14, "max_ms": 102.274, "p95_ms": 102.274}, "read_interval": {"avg_ms": 2376.487, "count": 14, "max_ms": 16301.165, "p95_ms": 16301.165}, "triton_wait": {"avg_ms": 79.071, "count": 14, "max_ms": 89.765, "p95_ms": 89.765}, "visualize": {"avg_ms": 3.811, "count": 14, "max_ms": 4.111, "p95_ms": 4.111}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.683, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3637}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4283.715, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 62, "h264_decode_error_count": 62, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:52:29+09:00", "last_error_summary": "[h264 @ 0x559e95d55180] error while decoding MB 5 65, bytestream -5 [in#0/rtsp @ 0x559e95ccde00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559e95ccde00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:52:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 155.795, "last_started_at": "2026-07-02T03:49:53+09:00", "pid": 36476, "process": {"exists": false, "pid": 36476}, "restart_count": 61, "restart_count_per_hour": 13, "rtsp_timeout_count": 112, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": false, "start_count": 62}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.592, "sample_count": 2190, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 436502, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:52:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-02T03:52:29+09:00", "last_frame_read_age_sec": 0.111, "last_frame_read_at": "2026-07-02T03:52:29+09:00", "mjpeg": {"age_sec": 0.106, "exists": true, "mtime": "2026-07-02T03:52:29.490425+09:00", "mtime_epoch": 1782931949.4904249, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 104769}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:52:29+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1078.0, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.791, "pushed": 0.791, "read": 0.791}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.424, "count": 14, "max_ms": 3.524, "p95_ms": 3.524}, "gstreamer_push": {"avg_ms": 1.54, "count": 14, "max_ms": 3.959, "p95_ms": 3.959}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 65.304, "count": 14, "max_ms": 77.777, "p95_ms": 77.777}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.011, "count": 14, "max_ms": 81.946, "p95_ms": 81.946}, "yolo11_640_fire_11class": {"avg_ms": 70.428, "count": 14, "max_ms": 78.882, "p95_ms": 78.882}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.875, "count": 14, "max_ms": 89.753, "p95_ms": 89.753}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.573, "count": 14, "max_ms": 102.274, "p95_ms": 102.274}, "read_interval": {"avg_ms": 2376.487, "count": 14, "max_ms": 16301.165, "p95_ms": 16301.165}, "triton_wait": {"avg_ms": 79.071, "count": 14, "max_ms": 89.765, "p95_ms": 89.765}, "visualize": {"avg_ms": 3.811, "count": 14, "max_ms": 4.111, "p95_ms": 4.111}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.69, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3637}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4298.988, "limit_mb": 8796093022207.996}, "detail": {"pid": 36628, "restart_count": 62, "start_count": 63}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 62, "h264_decode_error_count": 62, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:52:29+09:00", "last_error_summary": "[h264 @ 0x559e95d55180] error while decoding MB 5 65, bytestream -5 [in#0/rtsp @ 0x559e95ccde00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559e95ccde00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:52:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T03:52:31+09:00", "pid": 36628, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.281, "name": "ffmpeg", "pid": 36628, "rss_mb": 57.281, "state": "S (sleeping)", "threads": 1}, "restart_count": 62, "restart_count_per_hour": 14, "rtsp_timeout_count": 112, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": true, "start_count": 63}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.552, "sample_count": 2152, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 436502, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:52:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.194, "last_frame_pushed_at": "2026-07-02T03:52:29+09:00", "last_frame_read_age_sec": 4.293, "last_frame_read_at": "2026-07-02T03:52:29+09:00", "mjpeg": {"age_sec": 4.18, "exists": true, "mtime": "2026-07-02T03:52:29.598428+09:00", "mtime_epoch": 1782931949.5984275, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 95853}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:52:33+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1078.0, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.64, "pushed": 0.64, "read": 0.64}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.424, "count": 14, "max_ms": 3.524, "p95_ms": 3.524}, "gstreamer_push": {"avg_ms": 1.54, "count": 14, "max_ms": 3.959, "p95_ms": 3.959}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 65.304, "count": 14, "max_ms": 77.777, "p95_ms": 77.777}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.011, "count": 14, "max_ms": 81.946, "p95_ms": 81.946}, "yolo11_640_fire_11class": {"avg_ms": 70.428, "count": 14, "max_ms": 78.882, "p95_ms": 78.882}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.875, "count": 14, "max_ms": 89.753, "p95_ms": 89.753}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.573, "count": 14, "max_ms": 102.274, "p95_ms": 102.274}, "read_interval": {"avg_ms": 2376.487, "count": 14, "max_ms": 16301.165, "p95_ms": 16301.165}, "triton_wait": {"avg_ms": 79.071, "count": 14, "max_ms": 89.765, "p95_ms": 89.765}, "visualize": {"avg_ms": 3.811, "count": 14, "max_ms": 4.111, "p95_ms": 4.111}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.872, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3655}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4294.805, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x555898f7a080] error while decoding MB 116 33, bytestream -11 [in#0/rtsp @ 0x5558984fce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5558984fce00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 63, "h264_decode_error_count": 63, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:56:45+09:00", "last_error_summary": "[h264 @ 0x555898f7a080] error while decoding MB 116 33, bytestream -11 [in#0/rtsp @ 0x5558984fce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5558984fce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:52:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 253.866, "last_started_at": "2026-07-02T03:52:31+09:00", "pid": 36628, "process": {"exists": false, "pid": 36628}, "restart_count": 62, "restart_count_per_hour": 12, "rtsp_timeout_count": 114, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": false, "start_count": 63}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.345, "sample_count": 2217, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 438609, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:56:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.076, "last_frame_pushed_at": "2026-07-02T03:56:45+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-02T03:56:45+09:00", "mjpeg": {"age_sec": 0.063, "exists": true, "mtime": "2026-07-02T03:56:45.580347+09:00", "mtime_epoch": 1782932205.580347, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 80977}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:56:45+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1076.336, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 79, "pushed_frames": 79, "read_frames": 79, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.634, "pushed": 2.634, "read": 2.634}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.486, "count": 79, "max_ms": 4.478, "p95_ms": 3.968}, "gstreamer_push": {"avg_ms": 1.657, "count": 79, "max_ms": 6.166, "p95_ms": 3.801}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.623, "count": 79, "max_ms": 79.127, "p95_ms": 75.877}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.993, "count": 79, "max_ms": 83.204, "p95_ms": 80.865}, "yolo11_640_fire_11class": {"avg_ms": 71.165, "count": 79, "max_ms": 79.959, "p95_ms": 78.449}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.333, "count": 79, "max_ms": 89.367, "p95_ms": 86.943}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.08, "count": 79, "max_ms": 101.818, "p95_ms": 99.986}, "read_interval": {"avg_ms": 380.026, "count": 79, "max_ms": 12727.366, "p95_ms": 121.332}, "triton_wait": {"avg_ms": 78.441, "count": 79, "max_ms": 89.379, "p95_ms": 86.955}, "visualize": {"avg_ms": 3.754, "count": 79, "max_ms": 4.673, "p95_ms": 4.41}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.99, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3655}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4294.723, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T03:56:45+09:00", "last_error_summary": "[h264 @ 0x555898f7a080] error while decoding MB 116 33, bytestream -11 [in#0/rtsp @ 0x5558984fce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5558984fce00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 63, "h264_decode_error_count": 63, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:56:45+09:00", "last_error_summary": "[h264 @ 0x555898f7a080] error while decoding MB 116 33, bytestream -11 [in#0/rtsp @ 0x5558984fce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5558984fce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:56:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 253.885, "last_started_at": "2026-07-02T03:52:31+09:00", "pid": 36628, "process": {"exists": false, "pid": 36628}, "restart_count": 62, "restart_count_per_hour": 12, "rtsp_timeout_count": 114, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": false, "start_count": 63}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.345, "sample_count": 2218, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 438610, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:56:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T03:56:45+09:00", "last_frame_read_age_sec": 0.083, "last_frame_read_at": "2026-07-02T03:56:45+09:00", "mjpeg": {"age_sec": 0.083, "exists": true, "mtime": "2026-07-02T03:56:45.580347+09:00", "mtime_epoch": 1782932205.580347, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 80977}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:56:45+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1076.336, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 79, "pushed_frames": 79, "read_frames": 79, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.634, "pushed": 2.634, "read": 2.634}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.486, "count": 79, "max_ms": 4.478, "p95_ms": 3.968}, "gstreamer_push": {"avg_ms": 1.657, "count": 79, "max_ms": 6.166, "p95_ms": 3.801}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.406, "count": 79, "max_ms": 79.127, "p95_ms": 75.877}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.697, "count": 79, "max_ms": 83.204, "p95_ms": 80.865}, "yolo11_640_fire_11class": {"avg_ms": 70.928, "count": 79, "max_ms": 79.959, "p95_ms": 78.449}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.99, "count": 79, "max_ms": 89.367, "p95_ms": 86.943}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.804, "count": 79, "max_ms": 101.818, "p95_ms": 99.986}, "read_interval": {"avg_ms": 380.026, "count": 79, "max_ms": 12727.366, "p95_ms": 121.332}, "triton_wait": {"avg_ms": 78.097, "count": 79, "max_ms": 89.379, "p95_ms": 86.955}, "visualize": {"avg_ms": 3.781, "count": 79, "max_ms": 5.699, "p95_ms": 4.49}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3655}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4294.57, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 63, "h264_decode_error_count": 63, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:56:45+09:00", "last_error_summary": "[h264 @ 0x555898f7a080] error while decoding MB 116 33, bytestream -11 [in#0/rtsp @ 0x5558984fce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5558984fce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:56:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 253.893, "last_started_at": "2026-07-02T03:52:31+09:00", "pid": 36628, "process": {"exists": false, "pid": 36628}, "restart_count": 62, "restart_count_per_hour": 12, "rtsp_timeout_count": 114, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": false, "start_count": 63}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.345, "sample_count": 2218, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 438610, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:56:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-02T03:56:45+09:00", "last_frame_read_age_sec": 0.091, "last_frame_read_at": "2026-07-02T03:56:45+09:00", "mjpeg": {"age_sec": 0.091, "exists": true, "mtime": "2026-07-02T03:56:45.580347+09:00", "mtime_epoch": 1782932205.580347, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 80977}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:56:45+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1076.336, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 79, "pushed_frames": 79, "read_frames": 78, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.641, "pushed": 2.641, "read": 2.608}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.486, "count": 79, "max_ms": 4.478, "p95_ms": 3.968}, "gstreamer_push": {"avg_ms": 1.657, "count": 79, "max_ms": 6.166, "p95_ms": 3.801}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.406, "count": 79, "max_ms": 79.127, "p95_ms": 75.877}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.697, "count": 79, "max_ms": 83.204, "p95_ms": 80.865}, "yolo11_640_fire_11class": {"avg_ms": 70.928, "count": 79, "max_ms": 79.959, "p95_ms": 78.449}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.99, "count": 79, "max_ms": 89.367, "p95_ms": 86.943}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.804, "count": 79, "max_ms": 101.818, "p95_ms": 99.986}, "read_interval": {"avg_ms": 383.458, "count": 78, "max_ms": 12727.366, "p95_ms": 121.332}, "triton_wait": {"avg_ms": 78.097, "count": 79, "max_ms": 89.379, "p95_ms": 86.955}, "visualize": {"avg_ms": 3.781, "count": 79, "max_ms": 5.699, "p95_ms": 4.49}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.91, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3655}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4314.781, "limit_mb": 8796093022207.996}, "detail": {"pid": 36780, "restart_count": 63, "start_count": 64}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 63, "h264_decode_error_count": 63, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:56:45+09:00", "last_error_summary": "[h264 @ 0x555898f7a080] error while decoding MB 116 33, bytestream -11 [in#0/rtsp @ 0x5558984fce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5558984fce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:56:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T03:56:48+09:00", "pid": 36780, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.074, "name": "ffmpeg", "pid": 36780, "rss_mb": 57.074, "state": "S (sleeping)", "threads": 1}, "restart_count": 63, "restart_count_per_hour": 13, "rtsp_timeout_count": 114, "rtsp_timeout_count_per_hour": 26, "rtsp_tolerant_mode": false, "running": true, "start_count": 64}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.34, "sample_count": 2173, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 438610, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:56:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 5.028, "last_frame_pushed_at": "2026-07-02T03:56:45+09:00", "last_frame_read_age_sec": 5.105, "last_frame_read_at": "2026-07-02T03:56:45+09:00", "mjpeg": {"age_sec": 5.016, "exists": true, "mtime": "2026-07-02T03:56:45.668349+09:00", "mtime_epoch": 1782932205.668349, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 81473}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T03:56:50+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1076.336, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 34, "pushed_frames": 34, "read_frames": 34, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.133, "pushed": 1.133, "read": 1.133}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.499, "count": 34, "max_ms": 4.165, "p95_ms": 3.828}, "gstreamer_push": {"avg_ms": 1.66, "count": 34, "max_ms": 3.848, "p95_ms": 3.775}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.919, "count": 34, "max_ms": 79.127, "p95_ms": 76.279}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.896, "count": 34, "max_ms": 83.204, "p95_ms": 83.045}, "yolo11_640_fire_11class": {"avg_ms": 66.655, "count": 34, "max_ms": 79.959, "p95_ms": 78.449}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.188, "count": 34, "max_ms": 89.367, "p95_ms": 88.828}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.323, "count": 34, "max_ms": 101.818, "p95_ms": 101.248}, "read_interval": {"avg_ms": 735.329, "count": 34, "max_ms": 12727.366, "p95_ms": 9006.808}, "triton_wait": {"avg_ms": 72.381, "count": 34, "max_ms": 89.379, "p95_ms": 88.84}, "visualize": {"avg_ms": 3.855, "count": 34, "max_ms": 5.699, "p95_ms": 4.673}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2687}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4329.012, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55a88a4e63c0] left block unavailable for requested intra mode [h264 @ 0x55a88a4e63c0] error while decoding MB 0 24, bytestream 2282 [h264 @ 0x55a88a4e63c0] error while decoding MB 65 7, bytestream -7 [in#0/rtsp @ 0x55a889731e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a889731e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 205, "h264_decode_error_count": 372, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:57:21+09:00", "last_error_summary": "[h264 @ 0x55a88a4e63c0] left block unavailable for requested intra mode [h264 @ 0x55a88a4e63c0] error while decoding MB 0 24, bytestream 2282 [h264 @ 0x55a88a4e63c0] error while decoding MB 65 7, bytestream -7 [in#0/rtsp @ 0x55a889731e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a889731e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:49:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 449.267, "last_started_at": "2026-07-02T03:49:51+09:00", "pid": 36369, "process": {"exists": false, "pid": 36369}, "restart_count": 230, "restart_count_per_hour": 5, "rtsp_timeout_count": 94, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 231}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.175, "sample_count": 2718, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 322497, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:57:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.067, "last_frame_pushed_at": "2026-07-02T03:57:20+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-02T03:57:21+09:00", "mjpeg": {"age_sec": 0.054, "exists": true, "mtime": "2026-07-02T03:57:21.013155+09:00", "mtime_epoch": 1782932241.0131555, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 122297}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T03:57:21+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1017.879, "name": "celery", "pid": 237, "rss_mb": 1013.18, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 101, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.371, "pushed": 3.371, "read": 3.371}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.554, "count": 101, "max_ms": 2.105, "p95_ms": 1.711}, "gstreamer_push": {"avg_ms": 1.526, "count": 101, "max_ms": 4.17, "p95_ms": 2.382}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 64.488, "count": 101, "max_ms": 79.587, "p95_ms": 77.427}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.938, "count": 101, "max_ms": 83.981, "p95_ms": 82.24}, "yolo11_640_fire_11class": {"avg_ms": 70.908, "count": 101, "max_ms": 83.207, "p95_ms": 79.556}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.119, "count": 101, "max_ms": 88.966, "p95_ms": 88.165}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.719, "count": 101, "max_ms": 99.656, "p95_ms": 98.489}, "read_interval": {"avg_ms": 296.636, "count": 101, "max_ms": 19147.093, "p95_ms": 116.896}, "triton_wait": {"avg_ms": 80.257, "count": 101, "max_ms": 88.979, "p95_ms": 88.181}, "visualize": {"avg_ms": 3.701, "count": 101, "max_ms": 4.665, "p95_ms": 3.876}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.962, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2687}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4328.793, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T03:57:21+09:00", "last_error_summary": "[h264 @ 0x55a88a4e63c0] left block unavailable for requested intra mode [h264 @ 0x55a88a4e63c0] error while decoding MB 0 24, bytestream 2282 [h264 @ 0x55a88a4e63c0] error while decoding MB 65 7, bytestream -7 [in#0/rtsp @ 0x55a889731e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a889731e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 205, "h264_decode_error_count": 372, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:57:21+09:00", "last_error_summary": "[h264 @ 0x55a88a4e63c0] left block unavailable for requested intra mode [h264 @ 0x55a88a4e63c0] error while decoding MB 0 24, bytestream 2282 [h264 @ 0x55a88a4e63c0] error while decoding MB 65 7, bytestream -7 [in#0/rtsp @ 0x55a889731e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a889731e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:57:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 449.292, "last_started_at": "2026-07-02T03:49:51+09:00", "pid": 36369, "process": {"exists": false, "pid": 36369}, "restart_count": 230, "restart_count_per_hour": 5, "rtsp_timeout_count": 94, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 231}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.175, "sample_count": 2718, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 322498, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:57:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T03:57:21+09:00", "last_frame_read_age_sec": 0.08, "last_frame_read_at": "2026-07-02T03:57:21+09:00", "mjpeg": {"age_sec": 0.077, "exists": true, "mtime": "2026-07-02T03:57:21.013155+09:00", "mtime_epoch": 1782932241.0131555, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 122297}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T03:57:21+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1017.879, "name": "celery", "pid": 237, "rss_mb": 1013.18, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 101, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.402, "pushed": 3.402, "read": 3.368}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.595, "count": 102, "max_ms": 5.691, "p95_ms": 1.9}, "gstreamer_push": {"avg_ms": 1.525, "count": 102, "max_ms": 4.17, "p95_ms": 2.382}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 64.371, "count": 102, "max_ms": 79.587, "p95_ms": 77.427}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.747, "count": 102, "max_ms": 83.981, "p95_ms": 82.24}, "yolo11_640_fire_11class": {"avg_ms": 70.735, "count": 102, "max_ms": 83.207, "p95_ms": 79.556}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.877, "count": 102, "max_ms": 88.966, "p95_ms": 88.165}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.528, "count": 102, "max_ms": 99.656, "p95_ms": 98.489}, "read_interval": {"avg_ms": 296.636, "count": 101, "max_ms": 19147.093, "p95_ms": 116.896}, "triton_wait": {"avg_ms": 80.015, "count": 102, "max_ms": 88.979, "p95_ms": 88.181}, "visualize": {"avg_ms": 3.711, "count": 102, "max_ms": 4.76, "p95_ms": 3.973}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.985, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2687}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4328.785, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 205, "h264_decode_error_count": 372, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:57:21+09:00", "last_error_summary": "[h264 @ 0x55a88a4e63c0] left block unavailable for requested intra mode [h264 @ 0x55a88a4e63c0] error while decoding MB 0 24, bytestream 2282 [h264 @ 0x55a88a4e63c0] error while decoding MB 65 7, bytestream -7 [in#0/rtsp @ 0x55a889731e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a889731e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:57:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 449.3, "last_started_at": "2026-07-02T03:49:51+09:00", "pid": 36369, "process": {"exists": false, "pid": 36369}, "restart_count": 230, "restart_count_per_hour": 5, "rtsp_timeout_count": 94, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 231}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.175, "sample_count": 2718, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 322498, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:57:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-02T03:57:21+09:00", "last_frame_read_age_sec": 0.088, "last_frame_read_at": "2026-07-02T03:57:21+09:00", "mjpeg": {"age_sec": 0.085, "exists": true, "mtime": "2026-07-02T03:57:21.013155+09:00", "mtime_epoch": 1782932241.0131555, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 122297}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T03:57:21+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1017.879, "name": "celery", "pid": 237, "rss_mb": 1013.18, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 101, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.401, "pushed": 3.401, "read": 3.367}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.595, "count": 102, "max_ms": 5.691, "p95_ms": 1.9}, "gstreamer_push": {"avg_ms": 1.525, "count": 102, "max_ms": 4.17, "p95_ms": 2.382}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 64.371, "count": 102, "max_ms": 79.587, "p95_ms": 77.427}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.747, "count": 102, "max_ms": 83.981, "p95_ms": 82.24}, "yolo11_640_fire_11class": {"avg_ms": 70.735, "count": 102, "max_ms": 83.207, "p95_ms": 79.556}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.877, "count": 102, "max_ms": 88.966, "p95_ms": 88.165}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.528, "count": 102, "max_ms": 99.656, "p95_ms": 98.489}, "read_interval": {"avg_ms": 296.636, "count": 101, "max_ms": 19147.093, "p95_ms": 116.896}, "triton_wait": {"avg_ms": 80.015, "count": 102, "max_ms": 88.979, "p95_ms": 88.181}, "visualize": {"avg_ms": 3.711, "count": 102, "max_ms": 4.76, "p95_ms": 3.973}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2687}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4436.316, "limit_mb": 8796093022207.996}, "detail": {"pid": 36932, "restart_count": 231, "start_count": 232}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 205, "h264_decode_error_count": 372, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T03:57:21+09:00", "last_error_summary": "[h264 @ 0x55a88a4e63c0] left block unavailable for requested intra mode [h264 @ 0x55a88a4e63c0] error while decoding MB 0 24, bytestream 2282 [h264 @ 0x55a88a4e63c0] error while decoding MB 65 7, bytestream -7 [in#0/rtsp @ 0x55a889731e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a889731e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:57:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T03:57:23+09:00", "pid": 36932, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.922, "name": "ffmpeg", "pid": 36932, "rss_mb": 138.922, "state": "S (sleeping)", "threads": 106}, "restart_count": 231, "restart_count_per_hour": 6, "rtsp_timeout_count": 94, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 232}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.225, "sample_count": 2663, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 322498, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T03:57:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.202, "last_frame_pushed_at": "2026-07-02T03:57:21+09:00", "last_frame_read_age_sec": 4.276, "last_frame_read_at": "2026-07-02T03:57:21+09:00", "mjpeg": {"age_sec": 4.189, "exists": true, "mtime": "2026-07-02T03:57:21.097157+09:00", "mtime_epoch": 1782932241.0971575, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 119399}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T03:57:25+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1017.879, "name": "celery", "pid": 237, "rss_mb": 1013.18, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 63, "pushed_frames": 63, "read_frames": 62, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.105, "pushed": 2.105, "read": 2.071}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.626, "count": 63, "max_ms": 5.691, "p95_ms": 2.001}, "gstreamer_push": {"avg_ms": 1.454, "count": 63, "max_ms": 3.897, "p95_ms": 1.523}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.62, "count": 63, "max_ms": 79.587, "p95_ms": 77.766}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.255, "count": 63, "max_ms": 82.656, "p95_ms": 81.056}, "yolo11_640_fire_11class": {"avg_ms": 72.655, "count": 63, "max_ms": 83.207, "p95_ms": 79.775}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.963, "count": 63, "max_ms": 88.838, "p95_ms": 88.272}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.663, "count": 63, "max_ms": 99.656, "p95_ms": 98.768}, "read_interval": {"avg_ms": 415.295, "count": 62, "max_ms": 19147.093, "p95_ms": 117.034}, "triton_wait": {"avg_ms": 80.103, "count": 63, "max_ms": 88.851, "p95_ms": 88.284}, "visualize": {"avg_ms": 3.745, "count": 63, "max_ms": 4.76, "p95_ms": 4.34}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.932, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3674}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4282.57, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55b281446440] error while decoding MB 107 52, bytestream -5 [in#0/rtsp @ 0x55b2812cee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b2812cee00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 64, "h264_decode_error_count": 64, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:01:29+09:00", "last_error_summary": "[h264 @ 0x55b281446440] error while decoding MB 107 52, bytestream -5 [in#0/rtsp @ 0x55b2812cee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b2812cee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T03:56:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 280.67, "last_started_at": "2026-07-02T03:56:48+09:00", "pid": 36780, "process": {"exists": false, "pid": 36780}, "restart_count": 63, "restart_count_per_hour": 13, "rtsp_timeout_count": 116, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": false, "start_count": 64}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.175, "sample_count": 2373, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 440970, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:01:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.076, "last_frame_pushed_at": "2026-07-02T04:01:29+09:00", "last_frame_read_age_sec": 0.063, "last_frame_read_at": "2026-07-02T04:01:29+09:00", "mjpeg": {"age_sec": 0.061, "exists": true, "mtime": "2026-07-02T04:01:29.292448+09:00", "mtime_epoch": 1782932489.2924483, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 79809}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T04:01:29+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1077.227, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 70, "pushed_frames": 70, "read_frames": 70, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.34, "pushed": 2.34, "read": 2.34}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.337, "count": 70, "max_ms": 3.601, "p95_ms": 3.48}, "gstreamer_push": {"avg_ms": 1.487, "count": 70, "max_ms": 5.747, "p95_ms": 1.453}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 64.171, "count": 70, "max_ms": 77.178, "p95_ms": 75.921}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.419, "count": 70, "max_ms": 83.704, "p95_ms": 79.054}, "yolo11_640_fire_11class": {"avg_ms": 69.359, "count": 70, "max_ms": 77.917, "p95_ms": 76.193}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.465, "count": 70, "max_ms": 87.537, "p95_ms": 85.556}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.004, "count": 70, "max_ms": 98.711, "p95_ms": 97.331}, "read_interval": {"avg_ms": 427.694, "count": 70, "max_ms": 13751.245, "p95_ms": 116.139}, "triton_wait": {"avg_ms": 76.827, "count": 70, "max_ms": 87.548, "p95_ms": 85.566}, "visualize": {"avg_ms": 3.639, "count": 70, "max_ms": 4.089, "p95_ms": 3.906}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.912, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3674}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4282.461, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T04:01:29+09:00", "last_error_summary": "[h264 @ 0x55b281446440] error while decoding MB 107 52, bytestream -5 [in#0/rtsp @ 0x55b2812cee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b2812cee00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 64, "h264_decode_error_count": 64, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:01:29+09:00", "last_error_summary": "[h264 @ 0x55b281446440] error while decoding MB 107 52, bytestream -5 [in#0/rtsp @ 0x55b2812cee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b2812cee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T04:01:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 280.689, "last_started_at": "2026-07-02T03:56:48+09:00", "pid": 36780, "process": {"exists": false, "pid": 36780}, "restart_count": 63, "restart_count_per_hour": 13, "rtsp_timeout_count": 116, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": false, "start_count": 64}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.175, "sample_count": 2374, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 440971, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:01:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T04:01:29+09:00", "last_frame_read_age_sec": 0.082, "last_frame_read_at": "2026-07-02T04:01:29+09:00", "mjpeg": {"age_sec": 0.08, "exists": true, "mtime": "2026-07-02T04:01:29.292448+09:00", "mtime_epoch": 1782932489.2924483, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 79809}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T04:01:29+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1077.227, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 71, "pushed_frames": 71, "read_frames": 70, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.372, "pushed": 2.372, "read": 2.339}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.339, "count": 71, "max_ms": 3.601, "p95_ms": 3.481}, "gstreamer_push": {"avg_ms": 1.485, "count": 71, "max_ms": 5.747, "p95_ms": 1.453}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.904, "count": 71, "max_ms": 77.178, "p95_ms": 75.921}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.172, "count": 71, "max_ms": 83.704, "p95_ms": 79.054}, "yolo11_640_fire_11class": {"avg_ms": 69.131, "count": 71, "max_ms": 77.917, "p95_ms": 76.193}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.18, "count": 71, "max_ms": 87.537, "p95_ms": 85.556}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.79, "count": 71, "max_ms": 98.711, "p95_ms": 97.331}, "read_interval": {"avg_ms": 427.694, "count": 70, "max_ms": 13751.245, "p95_ms": 116.139}, "triton_wait": {"avg_ms": 76.536, "count": 71, "max_ms": 87.548, "p95_ms": 85.566}, "visualize": {"avg_ms": 3.681, "count": 71, "max_ms": 6.612, "p95_ms": 3.912}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.931, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3674}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4282.547, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 64, "h264_decode_error_count": 64, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:01:29+09:00", "last_error_summary": "[h264 @ 0x55b281446440] error while decoding MB 107 52, bytestream -5 [in#0/rtsp @ 0x55b2812cee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b2812cee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T04:01:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 280.697, "last_started_at": "2026-07-02T03:56:48+09:00", "pid": 36780, "process": {"exists": false, "pid": 36780}, "restart_count": 63, "restart_count_per_hour": 13, "rtsp_timeout_count": 116, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": false, "start_count": 64}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.175, "sample_count": 2374, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 440971, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:01:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-02T04:01:29+09:00", "last_frame_read_age_sec": 0.091, "last_frame_read_at": "2026-07-02T04:01:29+09:00", "mjpeg": {"age_sec": 0.089, "exists": true, "mtime": "2026-07-02T04:01:29.292448+09:00", "mtime_epoch": 1782932489.2924483, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 79809}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T04:01:29+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1077.227, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 71, "pushed_frames": 71, "read_frames": 70, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.371, "pushed": 2.371, "read": 2.338}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.339, "count": 71, "max_ms": 3.601, "p95_ms": 3.481}, "gstreamer_push": {"avg_ms": 1.485, "count": 71, "max_ms": 5.747, "p95_ms": 1.453}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.904, "count": 71, "max_ms": 77.178, "p95_ms": 75.921}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.172, "count": 71, "max_ms": 83.704, "p95_ms": 79.054}, "yolo11_640_fire_11class": {"avg_ms": 69.131, "count": 71, "max_ms": 77.917, "p95_ms": 76.193}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.18, "count": 71, "max_ms": 87.537, "p95_ms": 85.556}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.79, "count": 71, "max_ms": 98.711, "p95_ms": 97.331}, "read_interval": {"avg_ms": 427.694, "count": 70, "max_ms": 13751.245, "p95_ms": 116.139}, "triton_wait": {"avg_ms": 76.536, "count": 71, "max_ms": 87.548, "p95_ms": 85.566}, "visualize": {"avg_ms": 3.681, "count": 71, "max_ms": 6.612, "p95_ms": 3.912}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.94, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3674}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4403.48, "limit_mb": 8796093022207.996}, "detail": {"pid": 37039, "restart_count": 64, "start_count": 65}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 64, "h264_decode_error_count": 64, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:01:29+09:00", "last_error_summary": "[h264 @ 0x55b281446440] error while decoding MB 107 52, bytestream -5 [in#0/rtsp @ 0x55b2812cee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b2812cee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T04:01:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T04:01:32+09:00", "pid": 37039, "process": {"exists": true, "fd_count": 5, "hwm_mb": 156.961, "name": "ffmpeg", "pid": 37039, "rss_mb": 156.797, "state": "R (running)", "threads": 49}, "restart_count": 64, "restart_count_per_hour": 14, "rtsp_timeout_count": 116, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": true, "start_count": 65}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.175, "sample_count": 2373, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 440971, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:01:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.649, "last_frame_pushed_at": "2026-07-02T04:01:29+09:00", "last_frame_read_age_sec": 4.725, "last_frame_read_at": "2026-07-02T04:01:29+09:00", "mjpeg": {"age_sec": 4.635, "exists": true, "mtime": "2026-07-02T04:01:29.380447+09:00", "mtime_epoch": 1782932489.3804474, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 75135}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T04:01:34+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1077.227, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 29, "pushed_frames": 29, "read_frames": 28, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.969, "pushed": 0.969, "read": 0.936}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.244, "count": 29, "max_ms": 3.601, "p95_ms": 3.499}, "gstreamer_push": {"avg_ms": 1.327, "count": 29, "max_ms": 1.421, "p95_ms": 1.387}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.03, "count": 29, "max_ms": 75.921, "p95_ms": 73.378}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.941, "count": 29, "max_ms": 81.062, "p95_ms": 78.974}, "yolo11_640_fire_11class": {"avg_ms": 63.809, "count": 29, "max_ms": 77.917, "p95_ms": 77.761}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.623, "count": 29, "max_ms": 87.537, "p95_ms": 85.923}}, "truncated_count": 0}, "processing_total": {"avg_ms": 81.372, "count": 29, "max_ms": 98.711, "p95_ms": 98.004}, "read_interval": {"avg_ms": 903.463, "count": 28, "max_ms": 13751.245, "p95_ms": 9021.757}, "triton_wait": {"avg_ms": 69.277, "count": 29, "max_ms": 87.548, "p95_ms": 85.934}, "visualize": {"avg_ms": 3.688, "count": 29, "max_ms": 6.612, "p95_ms": 4.089}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.929, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3684}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4289.75, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55f8042afac0] error while decoding MB 107 33, bytestream -23 [in#0/rtsp @ 0x55f80385de00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f80385de00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 65, "h264_decode_error_count": 65, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:04:12+09:00", "last_error_summary": "[h264 @ 0x55f8042afac0] error while decoding MB 107 33, bytestream -23 [in#0/rtsp @ 0x55f80385de00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f80385de00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T04:01:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 160.3, "last_started_at": "2026-07-02T04:01:32+09:00", "pid": 37039, "process": {"exists": false, "pid": 37039}, "restart_count": 64, "restart_count_per_hour": 13, "rtsp_timeout_count": 118, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": false, "start_count": 65}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.164, "sample_count": 2209, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 442140, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:04:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.086, "last_frame_pushed_at": "2026-07-02T04:04:12+09:00", "last_frame_read_age_sec": 0.074, "last_frame_read_at": "2026-07-02T04:04:12+09:00", "mjpeg": {"age_sec": 0.071, "exists": true, "mtime": "2026-07-02T04:04:12.243316+09:00", "mtime_epoch": 1782932652.2433162, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 86311}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T04:04:12+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1080.52, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.909, "pushed": 8.909, "read": 9.651}, "hints": ["triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.18, "count": 12, "max_ms": 3.442, "p95_ms": 3.442}, "gstreamer_push": {"avg_ms": 1.344, "count": 12, "max_ms": 1.437, "p95_ms": 1.437}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.274, "count": 12, "max_ms": 70.291, "p95_ms": 70.291}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.812, "count": 12, "max_ms": 80.726, "p95_ms": 80.726}, "yolo11_640_fire_11class": {"avg_ms": 72.633, "count": 12, "max_ms": 78.214, "p95_ms": 78.214}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.481, "count": 12, "max_ms": 84.048, "p95_ms": 84.048}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.506, "count": 12, "max_ms": 96.143, "p95_ms": 96.143}, "read_interval": {"avg_ms": 2365.907, "count": 13, "max_ms": 29484.302, "p95_ms": 29484.302}, "triton_wait": {"avg_ms": 79.886, "count": 12, "max_ms": 84.06, "p95_ms": 84.06}, "visualize": {"avg_ms": 3.467, "count": 12, "max_ms": 3.742, "p95_ms": 3.742}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 1.347, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3684}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4289.434, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T04:04:12+09:00", "last_error_summary": "[h264 @ 0x55f8042afac0] error while decoding MB 107 33, bytestream -23 [in#0/rtsp @ 0x55f80385de00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f80385de00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 65, "h264_decode_error_count": 65, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:04:12+09:00", "last_error_summary": "[h264 @ 0x55f8042afac0] error while decoding MB 107 33, bytestream -23 [in#0/rtsp @ 0x55f80385de00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f80385de00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T04:04:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 160.33, "last_started_at": "2026-07-02T04:01:32+09:00", "pid": 37039, "process": {"exists": false, "pid": 37039}, "restart_count": 64, "restart_count_per_hour": 13, "rtsp_timeout_count": 118, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": false, "start_count": 65}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.164, "sample_count": 2210, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 442141, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:04:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T04:04:12+09:00", "last_frame_read_age_sec": 0.105, "last_frame_read_at": "2026-07-02T04:04:12+09:00", "mjpeg": {"age_sec": 0.102, "exists": true, "mtime": "2026-07-02T04:04:12.243316+09:00", "mtime_epoch": 1782932652.2433162, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 86311}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T04:04:12+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1080.52, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 9.438, "pushed": 9.438, "read": 9.438}, "hints": ["triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.191, "count": 13, "max_ms": 3.442, "p95_ms": 3.442}, "gstreamer_push": {"avg_ms": 1.343, "count": 13, "max_ms": 1.437, "p95_ms": 1.437}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.914, "count": 13, "max_ms": 70.291, "p95_ms": 70.291}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.974, "count": 13, "max_ms": 80.726, "p95_ms": 80.726}, "yolo11_640_fire_11class": {"avg_ms": 72.994, "count": 13, "max_ms": 78.214, "p95_ms": 78.214}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.914, "count": 13, "max_ms": 85.104, "p95_ms": 85.104}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.921, "count": 13, "max_ms": 96.899, "p95_ms": 96.899}, "read_interval": {"avg_ms": 2365.907, "count": 13, "max_ms": 29484.302, "p95_ms": 29484.302}, "triton_wait": {"avg_ms": 80.288, "count": 13, "max_ms": 85.116, "p95_ms": 85.116}, "visualize": {"avg_ms": 3.474, "count": 13, "max_ms": 3.742, "p95_ms": 3.742}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 1.377, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3684}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4289.227, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 65, "h264_decode_error_count": 65, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:04:12+09:00", "last_error_summary": "[h264 @ 0x55f8042afac0] error while decoding MB 107 33, bytestream -23 [in#0/rtsp @ 0x55f80385de00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f80385de00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T04:04:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 160.338, "last_started_at": "2026-07-02T04:01:32+09:00", "pid": 37039, "process": {"exists": false, "pid": 37039}, "restart_count": 64, "restart_count_per_hour": 13, "rtsp_timeout_count": 118, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": false, "start_count": 65}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.164, "sample_count": 2210, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 442141, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:04:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-02T04:04:12+09:00", "last_frame_read_age_sec": 0.112, "last_frame_read_at": "2026-07-02T04:04:12+09:00", "mjpeg": {"age_sec": 0.109, "exists": true, "mtime": "2026-07-02T04:04:12.243316+09:00", "mtime_epoch": 1782932652.2433162, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 86311}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T04:04:12+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1080.52, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 9.39, "pushed": 9.39, "read": 9.39}, "hints": ["triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.191, "count": 13, "max_ms": 3.442, "p95_ms": 3.442}, "gstreamer_push": {"avg_ms": 1.343, "count": 13, "max_ms": 1.437, "p95_ms": 1.437}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.914, "count": 13, "max_ms": 70.291, "p95_ms": 70.291}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.974, "count": 13, "max_ms": 80.726, "p95_ms": 80.726}, "yolo11_640_fire_11class": {"avg_ms": 72.994, "count": 13, "max_ms": 78.214, "p95_ms": 78.214}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.914, "count": 13, "max_ms": 85.104, "p95_ms": 85.104}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.921, "count": 13, "max_ms": 96.899, "p95_ms": 96.899}, "read_interval": {"avg_ms": 2365.907, "count": 13, "max_ms": 29484.302, "p95_ms": 29484.302}, "triton_wait": {"avg_ms": 80.288, "count": 13, "max_ms": 85.116, "p95_ms": 85.116}, "visualize": {"avg_ms": 3.474, "count": 13, "max_ms": 3.742, "p95_ms": 3.742}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 1.384, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3684}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4432.129, "limit_mb": 8796093022207.996}, "detail": {"pid": 37191, "restart_count": 65, "start_count": 66}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 65, "h264_decode_error_count": 65, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:04:12+09:00", "last_error_summary": "[h264 @ 0x55f8042afac0] error while decoding MB 107 33, bytestream -23 [in#0/rtsp @ 0x55f80385de00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f80385de00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T04:04:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T04:04:15+09:00", "pid": 37191, "process": {"exists": true, "fd_count": 5, "hwm_mb": 172.352, "name": "ffmpeg", "pid": 37191, "rss_mb": 172.352, "state": "R (running)", "threads": 106}, "restart_count": 65, "restart_count_per_hour": 14, "rtsp_timeout_count": 118, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": true, "start_count": 66}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.175, "sample_count": 2167, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 442141, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:04:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.748, "last_frame_pushed_at": "2026-07-02T04:04:12+09:00", "last_frame_read_age_sec": 4.847, "last_frame_read_at": "2026-07-02T04:04:12+09:00", "mjpeg": {"age_sec": 4.732, "exists": true, "mtime": "2026-07-02T04:04:12.355316+09:00", "mtime_epoch": 1782932652.3553157, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 82840}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T04:04:17+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1080.52, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.124, "pushed": 2.124, "read": 2.124}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.191, "count": 13, "max_ms": 3.442, "p95_ms": 3.442}, "gstreamer_push": {"avg_ms": 1.343, "count": 13, "max_ms": 1.437, "p95_ms": 1.437}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.914, "count": 13, "max_ms": 70.291, "p95_ms": 70.291}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.974, "count": 13, "max_ms": 80.726, "p95_ms": 80.726}, "yolo11_640_fire_11class": {"avg_ms": 72.994, "count": 13, "max_ms": 78.214, "p95_ms": 78.214}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.914, "count": 13, "max_ms": 85.104, "p95_ms": 85.104}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.921, "count": 13, "max_ms": 96.899, "p95_ms": 96.899}, "read_interval": {"avg_ms": 2365.907, "count": 13, "max_ms": 29484.302, "p95_ms": 29484.302}, "triton_wait": {"avg_ms": 80.288, "count": 13, "max_ms": 85.116, "p95_ms": 85.116}, "visualize": {"avg_ms": 3.474, "count": 13, "max_ms": 3.742, "p95_ms": 3.742}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 6.12, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2751}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4331.328, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 6, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55f4608ccb40] reference picture missing during reorder [h264 @ 0x55f4608ccb40] Missing reference picture, default is 65557 [h264 @ 0x55f4608e0d00] mmco: unref short failure [h264 @ 0x55f4608e0d00] illegal short term buffer state detected [h264 @ 0x55f4612de800] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f4612de800] error while decoding MB 0 21, bytestream 8955 [h264 @ 0x55f4607c7a40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f4607c7a40] er", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 206, "h264_decode_error_count": 378, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:11:26+09:00", "last_error_summary": "[h264 @ 0x55f4608ccb40] reference picture missing during reorder [h264 @ 0x55f4608ccb40] Missing reference picture, default is 65557 [h264 @ 0x55f4608e0d00] mmco: unref short failure [h264 @ 0x55f4608e0d00] illegal short term buffer state detected [h264 @ 0x55f4612de800] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f4612de800] error while decoding MB 0 21, bytestream 8955 [h264 @ 0x55f4607c7a40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f4607c7a40] er", "last_exit_at": "2026-07-02T03:57:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 842.898, "last_started_at": "2026-07-02T03:57:23+09:00", "pid": 36932, "process": {"exists": false, "pid": 36932}, "restart_count": 231, "restart_count_per_hour": 5, "rtsp_timeout_count": 96, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 232}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.468, "sample_count": 2541, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 330209, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:11:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.071, "last_frame_pushed_at": "2026-07-02T04:11:26+09:00", "last_frame_read_age_sec": 0.058, "last_frame_read_at": "2026-07-02T04:11:26+09:00", "mjpeg": {"age_sec": 0.055, "exists": true, "mtime": "2026-07-02T04:11:26.128896+09:00", "mtime_epoch": 1782933086.1288958, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 115653}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:11:26+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1021.746, "name": "celery", "pid": 237, "rss_mb": 1016.906, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 100, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.334, "pushed": 3.334, "read": 3.334}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.539, "count": 101, "max_ms": 1.873, "p95_ms": 1.569}, "gstreamer_push": {"avg_ms": 1.729, "count": 100, "max_ms": 6.256, "p95_ms": 3.938}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.626, "count": 101, "max_ms": 77.494, "p95_ms": 76.515}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.425, "count": 101, "max_ms": 82.627, "p95_ms": 80.398}, "yolo11_640_fire_11class": {"avg_ms": 74.147, "count": 101, "max_ms": 81.831, "p95_ms": 79.203}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.227, "count": 101, "max_ms": 89.391, "p95_ms": 86.011}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.057, "count": 100, "max_ms": 99.529, "p95_ms": 96.745}, "read_interval": {"avg_ms": 300.281, "count": 100, "max_ms": 19156.997, "p95_ms": 115.075}, "triton_wait": {"avg_ms": 82.307, "count": 101, "max_ms": 89.403, "p95_ms": 86.023}, "visualize": {"avg_ms": 3.674, "count": 101, "max_ms": 4.648, "p95_ms": 3.83}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2751}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4331.293, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T04:11:26+09:00", "last_error_summary": "[h264 @ 0x55f4608ccb40] reference picture missing during reorder [h264 @ 0x55f4608ccb40] Missing reference picture, default is 65557 [h264 @ 0x55f4608e0d00] mmco: unref short failure [h264 @ 0x55f4608e0d00] illegal short term buffer state detected [h264 @ 0x55f4612de800] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f4612de800] error while decoding MB 0 21, bytestream 8955 [h264 @ 0x55f4607c7a40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f4607c7a40] er", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 206, "h264_decode_error_count": 378, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:11:26+09:00", "last_error_summary": "[h264 @ 0x55f4608ccb40] reference picture missing during reorder [h264 @ 0x55f4608ccb40] Missing reference picture, default is 65557 [h264 @ 0x55f4608e0d00] mmco: unref short failure [h264 @ 0x55f4608e0d00] illegal short term buffer state detected [h264 @ 0x55f4612de800] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f4612de800] error while decoding MB 0 21, bytestream 8955 [h264 @ 0x55f4607c7a40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f4607c7a40] er", "last_exit_at": "2026-07-02T04:11:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 842.916, "last_started_at": "2026-07-02T03:57:23+09:00", "pid": 36932, "process": {"exists": false, "pid": 36932}, "restart_count": 231, "restart_count_per_hour": 5, "rtsp_timeout_count": 96, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 232}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.468, "sample_count": 2541, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 330209, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:11:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T04:11:26+09:00", "last_frame_read_age_sec": 0.077, "last_frame_read_at": "2026-07-02T04:11:26+09:00", "mjpeg": {"age_sec": 0.073, "exists": true, "mtime": "2026-07-02T04:11:26.128896+09:00", "mtime_epoch": 1782933086.1288958, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 115653}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:11:26+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1021.746, "name": "celery", "pid": 237, "rss_mb": 1016.906, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 100, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.334, "pushed": 3.334, "read": 3.334}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.539, "count": 100, "max_ms": 1.873, "p95_ms": 1.569}, "gstreamer_push": {"avg_ms": 1.729, "count": 100, "max_ms": 6.256, "p95_ms": 3.938}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.64, "count": 100, "max_ms": 77.494, "p95_ms": 76.515}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.389, "count": 100, "max_ms": 82.627, "p95_ms": 80.398}, "yolo11_640_fire_11class": {"avg_ms": 74.128, "count": 100, "max_ms": 81.831, "p95_ms": 79.203}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.203, "count": 100, "max_ms": 89.391, "p95_ms": 86.011}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.787, "count": 100, "max_ms": 99.529, "p95_ms": 96.745}, "read_interval": {"avg_ms": 300.281, "count": 100, "max_ms": 19156.997, "p95_ms": 115.075}, "triton_wait": {"avg_ms": 82.283, "count": 100, "max_ms": 89.403, "p95_ms": 86.023}, "visualize": {"avg_ms": 3.674, "count": 100, "max_ms": 4.648, "p95_ms": 3.83}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.99, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2751}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4331.254, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 206, "h264_decode_error_count": 378, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:11:26+09:00", "last_error_summary": "[h264 @ 0x55f4608ccb40] reference picture missing during reorder [h264 @ 0x55f4608ccb40] Missing reference picture, default is 65557 [h264 @ 0x55f4608e0d00] mmco: unref short failure [h264 @ 0x55f4608e0d00] illegal short term buffer state detected [h264 @ 0x55f4612de800] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f4612de800] error while decoding MB 0 21, bytestream 8955 [h264 @ 0x55f4607c7a40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f4607c7a40] er", "last_exit_at": "2026-07-02T04:11:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 842.925, "last_started_at": "2026-07-02T03:57:23+09:00", "pid": 36932, "process": {"exists": false, "pid": 36932}, "restart_count": 231, "restart_count_per_hour": 5, "rtsp_timeout_count": 96, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 232}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.468, "sample_count": 2541, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 330209, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:11:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-02T04:11:26+09:00", "last_frame_read_age_sec": 0.085, "last_frame_read_at": "2026-07-02T04:11:26+09:00", "mjpeg": {"age_sec": 0.081, "exists": true, "mtime": "2026-07-02T04:11:26.128896+09:00", "mtime_epoch": 1782933086.1288958, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 115653}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:11:26+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1021.746, "name": "celery", "pid": 237, "rss_mb": 1016.906, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 100, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.333, "pushed": 3.333, "read": 3.333}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.539, "count": 100, "max_ms": 1.873, "p95_ms": 1.569}, "gstreamer_push": {"avg_ms": 1.729, "count": 100, "max_ms": 6.256, "p95_ms": 3.938}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.64, "count": 100, "max_ms": 77.494, "p95_ms": 76.515}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.389, "count": 100, "max_ms": 82.627, "p95_ms": 80.398}, "yolo11_640_fire_11class": {"avg_ms": 74.128, "count": 100, "max_ms": 81.831, "p95_ms": 79.203}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.203, "count": 100, "max_ms": 89.391, "p95_ms": 86.011}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.787, "count": 100, "max_ms": 99.529, "p95_ms": 96.745}, "read_interval": {"avg_ms": 300.281, "count": 100, "max_ms": 19156.997, "p95_ms": 115.075}, "triton_wait": {"avg_ms": 82.283, "count": 100, "max_ms": 89.403, "p95_ms": 86.023}, "visualize": {"avg_ms": 3.674, "count": 100, "max_ms": 4.648, "p95_ms": 3.83}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2751}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4441.738, "limit_mb": 8796093022207.996}, "detail": {"pid": 37343, "restart_count": 232, "start_count": 233}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 206, "h264_decode_error_count": 378, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:11:26+09:00", "last_error_summary": "[h264 @ 0x55f4608ccb40] reference picture missing during reorder [h264 @ 0x55f4608ccb40] Missing reference picture, default is 65557 [h264 @ 0x55f4608e0d00] mmco: unref short failure [h264 @ 0x55f4608e0d00] illegal short term buffer state detected [h264 @ 0x55f4612de800] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f4612de800] error while decoding MB 0 21, bytestream 8955 [h264 @ 0x55f4607c7a40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55f4607c7a40] er", "last_exit_at": "2026-07-02T04:11:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T04:11:28+09:00", "pid": 37343, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.18, "name": "ffmpeg", "pid": 37343, "rss_mb": 139.18, "state": "S (sleeping)", "threads": 106}, "restart_count": 232, "restart_count_per_hour": 6, "rtsp_timeout_count": 96, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 233}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.468, "sample_count": 2503, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 330209, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:11:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.16, "last_frame_pushed_at": "2026-07-02T04:11:26+09:00", "last_frame_read_age_sec": 4.23, "last_frame_read_at": "2026-07-02T04:11:26+09:00", "mjpeg": {"age_sec": 4.146, "exists": true, "mtime": "2026-07-02T04:11:26.208896+09:00", "mtime_epoch": 1782933086.2088964, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 115500}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:11:30+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1021.746, "name": "celery", "pid": 237, "rss_mb": 1016.906, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 63, "pushed_frames": 63, "read_frames": 62, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.104, "pushed": 2.104, "read": 2.071}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.539, "count": 63, "max_ms": 1.873, "p95_ms": 1.569}, "gstreamer_push": {"avg_ms": 1.77, "count": 63, "max_ms": 6.256, "p95_ms": 3.938}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.054, "count": 63, "max_ms": 77.014, "p95_ms": 76.297}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.174, "count": 63, "max_ms": 80.398, "p95_ms": 78.199}, "yolo11_640_fire_11class": {"avg_ms": 72.462, "count": 63, "max_ms": 80.648, "p95_ms": 79.006}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 81.137, "count": 63, "max_ms": 89.391, "p95_ms": 86.003}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.819, "count": 63, "max_ms": 99.529, "p95_ms": 96.985}, "read_interval": {"avg_ms": 416.173, "count": 62, "max_ms": 19156.997, "p95_ms": 115.179}, "triton_wait": {"avg_ms": 81.258, "count": 63, "max_ms": 89.403, "p95_ms": 86.015}, "visualize": {"avg_ms": 3.673, "count": 63, "max_ms": 4.147, "p95_ms": 3.871}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.937, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3722}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4275.508, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x55ae7d5d4200] missing picture in access unit with size 18 [h264 @ 0x55ae7e290f80] no frame! [in#0/rtsp @ 0x55ae7d5cfe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ae7d5cfe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55ae7d5d4840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 66, "h264_decode_error_count": 66, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:12:42+09:00", "last_error_summary": "[NULL @ 0x55ae7d5d4200] missing picture in access unit with size 18 [h264 @ 0x55ae7e290f80] no frame! [in#0/rtsp @ 0x55ae7d5cfe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ae7d5cfe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55ae7d5d4840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T04:04:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 506.921, "last_started_at": "2026-07-02T04:04:15+09:00", "pid": 37191, "process": {"exists": false, "pid": 37191}, "restart_count": 65, "restart_count_per_hour": 13, "rtsp_timeout_count": 120, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": false, "start_count": 66}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.876, "sample_count": 2654, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 446645, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:12:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.079, "last_frame_pushed_at": "2026-07-02T04:12:41+09:00", "last_frame_read_age_sec": 0.063, "last_frame_read_at": "2026-07-02T04:12:41+09:00", "mjpeg": {"age_sec": 0.066, "exists": true, "mtime": "2026-07-02T04:12:41.941612+09:00", "mtime_epoch": 1782933161.941612, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 79398}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T04:12:42+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1077.164, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 100, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.342, "pushed": 3.342, "read": 3.342}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.516, "count": 100, "max_ms": 5.242, "p95_ms": 4.236}, "gstreamer_push": {"avg_ms": 1.628, "count": 100, "max_ms": 4.666, "p95_ms": 3.809}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 67.329, "count": 100, "max_ms": 76.779, "p95_ms": 74.112}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.627, "count": 100, "max_ms": 83.2, "p95_ms": 79.501}, "yolo11_640_fire_11class": {"avg_ms": 71.388, "count": 100, "max_ms": 76.756, "p95_ms": 75.685}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.528, "count": 100, "max_ms": 89.744, "p95_ms": 84.904}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.388, "count": 100, "max_ms": 101.835, "p95_ms": 97.135}, "read_interval": {"avg_ms": 299.512, "count": 100, "max_ms": 10169.56, "p95_ms": 114.865}, "triton_wait": {"avg_ms": 79.712, "count": 100, "max_ms": 89.757, "p95_ms": 84.916}, "visualize": {"avg_ms": 3.78, "count": 100, "max_ms": 4.593, "p95_ms": 4.272}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.923, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3722}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4275.137, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T04:12:42+09:00", "last_error_summary": "[NULL @ 0x55ae7d5d4200] missing picture in access unit with size 18 [h264 @ 0x55ae7e290f80] no frame! [in#0/rtsp @ 0x55ae7d5cfe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ae7d5cfe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55ae7d5d4840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 66, "h264_decode_error_count": 66, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:12:42+09:00", "last_error_summary": "[NULL @ 0x55ae7d5d4200] missing picture in access unit with size 18 [h264 @ 0x55ae7e290f80] no frame! [in#0/rtsp @ 0x55ae7d5cfe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ae7d5cfe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55ae7d5d4840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T04:12:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 506.953, "last_started_at": "2026-07-02T04:04:15+09:00", "pid": 37191, "process": {"exists": false, "pid": 37191}, "restart_count": 65, "restart_count_per_hour": 13, "rtsp_timeout_count": 120, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": false, "start_count": 66}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.876, "sample_count": 2655, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 446646, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:12:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T04:12:42+09:00", "last_frame_read_age_sec": 0.095, "last_frame_read_at": "2026-07-02T04:12:41+09:00", "mjpeg": {"age_sec": 0.098, "exists": true, "mtime": "2026-07-02T04:12:41.941612+09:00", "mtime_epoch": 1782933161.941612, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 79398}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T04:12:42+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1077.164, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 100, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.372, "pushed": 3.372, "read": 3.338}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.512, "count": 101, "max_ms": 5.242, "p95_ms": 4.236}, "gstreamer_push": {"avg_ms": 1.625, "count": 101, "max_ms": 4.666, "p95_ms": 3.809}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 67.329, "count": 101, "max_ms": 76.779, "p95_ms": 74.112}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.619, "count": 101, "max_ms": 83.2, "p95_ms": 79.501}, "yolo11_640_fire_11class": {"avg_ms": 71.356, "count": 101, "max_ms": 76.756, "p95_ms": 75.685}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.476, "count": 101, "max_ms": 89.744, "p95_ms": 84.904}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.325, "count": 101, "max_ms": 101.835, "p95_ms": 97.135}, "read_interval": {"avg_ms": 299.512, "count": 100, "max_ms": 10169.56, "p95_ms": 114.865}, "triton_wait": {"avg_ms": 79.659, "count": 101, "max_ms": 89.757, "p95_ms": 84.916}, "visualize": {"avg_ms": 3.778, "count": 101, "max_ms": 4.593, "p95_ms": 4.272}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.954, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3722}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4275.0, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 66, "h264_decode_error_count": 66, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:12:42+09:00", "last_error_summary": "[NULL @ 0x55ae7d5d4200] missing picture in access unit with size 18 [h264 @ 0x55ae7e290f80] no frame! [in#0/rtsp @ 0x55ae7d5cfe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ae7d5cfe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55ae7d5d4840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T04:12:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 506.961, "last_started_at": "2026-07-02T04:04:15+09:00", "pid": 37191, "process": {"exists": false, "pid": 37191}, "restart_count": 65, "restart_count_per_hour": 13, "rtsp_timeout_count": 120, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": false, "start_count": 66}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.876, "sample_count": 2655, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 446646, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:12:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-02T04:12:42+09:00", "last_frame_read_age_sec": 0.103, "last_frame_read_at": "2026-07-02T04:12:41+09:00", "mjpeg": {"age_sec": 0.106, "exists": true, "mtime": "2026-07-02T04:12:41.941612+09:00", "mtime_epoch": 1782933161.941612, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 79398}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T04:12:42+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1077.164, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 100, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.371, "pushed": 3.371, "read": 3.338}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.512, "count": 101, "max_ms": 5.242, "p95_ms": 4.236}, "gstreamer_push": {"avg_ms": 1.625, "count": 101, "max_ms": 4.666, "p95_ms": 3.809}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 67.329, "count": 101, "max_ms": 76.779, "p95_ms": 74.112}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.619, "count": 101, "max_ms": 83.2, "p95_ms": 79.501}, "yolo11_640_fire_11class": {"avg_ms": 71.356, "count": 101, "max_ms": 76.756, "p95_ms": 75.685}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.476, "count": 101, "max_ms": 89.744, "p95_ms": 84.904}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.325, "count": 101, "max_ms": 101.835, "p95_ms": 97.135}, "read_interval": {"avg_ms": 299.512, "count": 100, "max_ms": 10169.56, "p95_ms": 114.865}, "triton_wait": {"avg_ms": 79.659, "count": 101, "max_ms": 89.757, "p95_ms": 84.916}, "visualize": {"avg_ms": 3.778, "count": 101, "max_ms": 4.593, "p95_ms": 4.272}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.962, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3722}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4295.496, "limit_mb": 8796093022207.996}, "detail": {"pid": 37450, "restart_count": 66, "start_count": 67}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 66, "h264_decode_error_count": 66, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:12:42+09:00", "last_error_summary": "[NULL @ 0x55ae7d5d4200] missing picture in access unit with size 18 [h264 @ 0x55ae7e290f80] no frame! [in#0/rtsp @ 0x55ae7d5cfe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ae7d5cfe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x55ae7d5d4840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T04:12:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T04:12:44+09:00", "pid": 37450, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.953, "name": "ffmpeg", "pid": 37450, "rss_mb": 57.953, "state": "S (sleeping)", "threads": 1}, "restart_count": 66, "restart_count_per_hour": 14, "rtsp_timeout_count": 120, "rtsp_timeout_count_per_hour": 28, "rtsp_tolerant_mode": false, "running": true, "start_count": 67}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.881, "sample_count": 2613, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 446646, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:12:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.619, "last_frame_pushed_at": "2026-07-02T04:12:42+09:00", "last_frame_read_age_sec": 4.707, "last_frame_read_at": "2026-07-02T04:12:41+09:00", "mjpeg": {"age_sec": 4.607, "exists": true, "mtime": "2026-07-02T04:12:42.045613+09:00", "mtime_epoch": 1782933162.045613, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 80308}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T04:12:46+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1077.164, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 59, "pushed_frames": 59, "read_frames": 58, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.969, "pushed": 1.969, "read": 1.935}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.464, "count": 59, "max_ms": 4.962, "p95_ms": 3.694}, "gstreamer_push": {"avg_ms": 1.637, "count": 59, "max_ms": 4.666, "p95_ms": 3.914}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 66.899, "count": 59, "max_ms": 76.779, "p95_ms": 75.579}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.825, "count": 59, "max_ms": 83.2, "p95_ms": 82.354}, "yolo11_640_fire_11class": {"avg_ms": 71.152, "count": 59, "max_ms": 76.079, "p95_ms": 75.726}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.684, "count": 59, "max_ms": 89.744, "p95_ms": 86.204}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.351, "count": 59, "max_ms": 101.835, "p95_ms": 98.395}, "read_interval": {"avg_ms": 436.996, "count": 58, "max_ms": 10169.56, "p95_ms": 115.968}, "triton_wait": {"avg_ms": 79.831, "count": 59, "max_ms": 89.757, "p95_ms": 86.217}, "visualize": {"avg_ms": 3.71, "count": 59, "max_ms": 4.47, "p95_ms": 4.074}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.969, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2780}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4338.875, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 5, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55c722008380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c722008380] error while decoding MB 0 28, bytestream 3279 [h264 @ 0x55c722967100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c722967100] error while decoding MB 0 24, bytestream 19378 [h264 @ 0x55c721f6bb80] cabac decode of qscale diff failed at 0 5 [h264 @ 0x55c721f6bb80] error while decoding MB 0 5, bytestream 518 [h264 @ 0x55c722a35f80] error while decoding MB 32 7, bytestream -8 [", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 207, "h264_decode_error_count": 383, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:17:57+09:00", "last_error_summary": "[h264 @ 0x55c722008380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c722008380] error while decoding MB 0 28, bytestream 3279 [h264 @ 0x55c722967100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c722967100] error while decoding MB 0 24, bytestream 19378 [h264 @ 0x55c721f6bb80] cabac decode of qscale diff failed at 0 5 [h264 @ 0x55c721f6bb80] error while decoding MB 0 5, bytestream 518 [h264 @ 0x55c722a35f80] error while decoding MB 32 7, bytestream -8 [", "last_exit_at": "2026-07-02T04:11:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 389.453, "last_started_at": "2026-07-02T04:11:28+09:00", "pid": 37343, "process": {"exists": false, "pid": 37343}, "restart_count": 232, "restart_count_per_hour": 5, "rtsp_timeout_count": 98, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 233}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.53, "sample_count": 2593, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 333673, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:17:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.102, "last_frame_pushed_at": "2026-07-02T04:17:57+09:00", "last_frame_read_age_sec": 0.086, "last_frame_read_at": "2026-07-02T04:17:57+09:00", "mjpeg": {"age_sec": 0.086, "exists": true, "mtime": "2026-07-02T04:17:57.721409+09:00", "mtime_epoch": 1782933477.721409, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 113870}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:17:57+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1023.094, "name": "celery", "pid": 237, "rss_mb": 1018.652, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 126, "pushed_frames": 127, "read_frames": 127, "select_timeouts": 14, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.2, "pushed": 4.234, "read": 4.234}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.519, "count": 126, "max_ms": 1.871, "p95_ms": 1.555}, "gstreamer_push": {"avg_ms": 1.462, "count": 127, "max_ms": 6.051, "p95_ms": 1.469}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.79, "count": 126, "max_ms": 79.985, "p95_ms": 69.084}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.014, "count": 126, "max_ms": 80.475, "p95_ms": 78.383}, "yolo11_640_fire_11class": {"avg_ms": 72.354, "count": 126, "max_ms": 80.069, "p95_ms": 77.353}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.685, "count": 126, "max_ms": 88.391, "p95_ms": 84.229}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.154, "count": 126, "max_ms": 98.681, "p95_ms": 94.748}, "read_interval": {"avg_ms": 236.267, "count": 127, "max_ms": 10086.335, "p95_ms": 111.733}, "triton_wait": {"avg_ms": 78.874, "count": 126, "max_ms": 88.403, "p95_ms": 84.241}, "visualize": {"avg_ms": 3.675, "count": 126, "max_ms": 4.136, "p95_ms": 3.883}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2780}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4338.293, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T04:17:57+09:00", "last_error_summary": "[h264 @ 0x55c722008380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c722008380] error while decoding MB 0 28, bytestream 3279 [h264 @ 0x55c722967100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c722967100] error while decoding MB 0 24, bytestream 19378 [h264 @ 0x55c721f6bb80] cabac decode of qscale diff failed at 0 5 [h264 @ 0x55c721f6bb80] error while decoding MB 0 5, bytestream 518 [h264 @ 0x55c722a35f80] error while decoding MB 32 7, bytestream -8 [", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 207, "h264_decode_error_count": 383, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:17:57+09:00", "last_error_summary": "[h264 @ 0x55c722008380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c722008380] error while decoding MB 0 28, bytestream 3279 [h264 @ 0x55c722967100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c722967100] error while decoding MB 0 24, bytestream 19378 [h264 @ 0x55c721f6bb80] cabac decode of qscale diff failed at 0 5 [h264 @ 0x55c721f6bb80] error while decoding MB 0 5, bytestream 518 [h264 @ 0x55c722a35f80] error while decoding MB 32 7, bytestream -8 [", "last_exit_at": "2026-07-02T04:17:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 389.473, "last_started_at": "2026-07-02T04:11:28+09:00", "pid": 37343, "process": {"exists": false, "pid": 37343}, "restart_count": 232, "restart_count_per_hour": 5, "rtsp_timeout_count": 98, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 233}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.53, "sample_count": 2594, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 333674, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:17:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T04:17:57+09:00", "last_frame_read_age_sec": 0.106, "last_frame_read_at": "2026-07-02T04:17:57+09:00", "mjpeg": {"age_sec": 0.106, "exists": true, "mtime": "2026-07-02T04:17:57.721409+09:00", "mtime_epoch": 1782933477.721409, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 113870}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:17:57+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1023.094, "name": "celery", "pid": 237, "rss_mb": 1018.652, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 127, "pushed_frames": 127, "read_frames": 127, "select_timeouts": 14, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.357, "pushed": 4.357, "read": 4.357}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.519, "count": 127, "max_ms": 1.871, "p95_ms": 1.555}, "gstreamer_push": {"avg_ms": 1.462, "count": 127, "max_ms": 6.051, "p95_ms": 1.469}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.899, "count": 127, "max_ms": 79.985, "p95_ms": 69.084}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.032, "count": 127, "max_ms": 80.475, "p95_ms": 78.383}, "yolo11_640_fire_11class": {"avg_ms": 72.364, "count": 127, "max_ms": 80.069, "p95_ms": 77.353}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.702, "count": 127, "max_ms": 88.391, "p95_ms": 84.229}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.213, "count": 127, "max_ms": 98.681, "p95_ms": 94.842}, "read_interval": {"avg_ms": 236.267, "count": 127, "max_ms": 10086.335, "p95_ms": 111.733}, "triton_wait": {"avg_ms": 78.889, "count": 127, "max_ms": 88.403, "p95_ms": 84.241}, "visualize": {"avg_ms": 3.699, "count": 127, "max_ms": 6.733, "p95_ms": 3.905}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.151, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2780}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4338.219, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 207, "h264_decode_error_count": 383, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:17:57+09:00", "last_error_summary": "[h264 @ 0x55c722008380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c722008380] error while decoding MB 0 28, bytestream 3279 [h264 @ 0x55c722967100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c722967100] error while decoding MB 0 24, bytestream 19378 [h264 @ 0x55c721f6bb80] cabac decode of qscale diff failed at 0 5 [h264 @ 0x55c721f6bb80] error while decoding MB 0 5, bytestream 518 [h264 @ 0x55c722a35f80] error while decoding MB 32 7, bytestream -8 [", "last_exit_at": "2026-07-02T04:17:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 389.482, "last_started_at": "2026-07-02T04:11:28+09:00", "pid": 37343, "process": {"exists": false, "pid": 37343}, "restart_count": 232, "restart_count_per_hour": 5, "rtsp_timeout_count": 98, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 233}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.53, "sample_count": 2594, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 333674, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:17:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-02T04:17:57+09:00", "last_frame_read_age_sec": 0.114, "last_frame_read_at": "2026-07-02T04:17:57+09:00", "mjpeg": {"age_sec": 0.115, "exists": true, "mtime": "2026-07-02T04:17:57.721409+09:00", "mtime_epoch": 1782933477.721409, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 113870}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:17:57+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1023.094, "name": "celery", "pid": 237, "rss_mb": 1018.652, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 127, "pushed_frames": 127, "read_frames": 127, "select_timeouts": 14, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.355, "pushed": 4.355, "read": 4.355}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.519, "count": 127, "max_ms": 1.871, "p95_ms": 1.555}, "gstreamer_push": {"avg_ms": 1.462, "count": 127, "max_ms": 6.051, "p95_ms": 1.469}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.899, "count": 127, "max_ms": 79.985, "p95_ms": 69.084}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.032, "count": 127, "max_ms": 80.475, "p95_ms": 78.383}, "yolo11_640_fire_11class": {"avg_ms": 72.364, "count": 127, "max_ms": 80.069, "p95_ms": 77.353}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.702, "count": 127, "max_ms": 88.391, "p95_ms": 84.229}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.213, "count": 127, "max_ms": 98.681, "p95_ms": 94.842}, "read_interval": {"avg_ms": 236.267, "count": 127, "max_ms": 10086.335, "p95_ms": 111.733}, "triton_wait": {"avg_ms": 78.889, "count": 127, "max_ms": 88.403, "p95_ms": 84.241}, "visualize": {"avg_ms": 3.699, "count": 127, "max_ms": 6.733, "p95_ms": 3.905}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.16, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2780}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4446.039, "limit_mb": 8796093022207.996}, "detail": {"pid": 37602, "restart_count": 233, "start_count": 234}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 207, "h264_decode_error_count": 383, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:17:57+09:00", "last_error_summary": "[h264 @ 0x55c722008380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c722008380] error while decoding MB 0 28, bytestream 3279 [h264 @ 0x55c722967100] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c722967100] error while decoding MB 0 24, bytestream 19378 [h264 @ 0x55c721f6bb80] cabac decode of qscale diff failed at 0 5 [h264 @ 0x55c721f6bb80] error while decoding MB 0 5, bytestream 518 [h264 @ 0x55c722a35f80] error while decoding MB 32 7, bytestream -8 [", "last_exit_at": "2026-07-02T04:18:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T04:18:00+09:00", "pid": 37602, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.207, "name": "ffmpeg", "pid": 37602, "rss_mb": 139.207, "state": "S (sleeping)", "threads": 106}, "restart_count": 233, "restart_count_per_hour": 6, "rtsp_timeout_count": 98, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 234}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.551, "sample_count": 2554, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 333674, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:17:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.399, "last_frame_pushed_at": "2026-07-02T04:17:57+09:00", "last_frame_read_age_sec": 4.498, "last_frame_read_at": "2026-07-02T04:17:57+09:00", "mjpeg": {"age_sec": 4.386, "exists": true, "mtime": "2026-07-02T04:17:57.833411+09:00", "mtime_epoch": 1782933477.8334107, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 112578}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:18:02+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1023.094, "name": "celery", "pid": 237, "rss_mb": 1018.652, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 125, "pushed_frames": 125, "read_frames": 125, "select_timeouts": 11, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.349, "pushed": 5.349, "read": 5.349}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.519, "count": 125, "max_ms": 1.871, "p95_ms": 1.555}, "gstreamer_push": {"avg_ms": 1.464, "count": 125, "max_ms": 6.051, "p95_ms": 1.469}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.926, "count": 125, "max_ms": 79.985, "p95_ms": 69.084}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.316, "count": 125, "max_ms": 80.475, "p95_ms": 78.383}, "yolo11_640_fire_11class": {"avg_ms": 72.627, "count": 125, "max_ms": 80.069, "p95_ms": 77.353}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.053, "count": 125, "max_ms": 88.391, "p95_ms": 84.229}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.563, "count": 125, "max_ms": 98.681, "p95_ms": 94.842}, "read_interval": {"avg_ms": 231.649, "count": 125, "max_ms": 10086.335, "p95_ms": 111.606}, "triton_wait": {"avg_ms": 79.239, "count": 125, "max_ms": 88.403, "p95_ms": 84.241}, "visualize": {"avg_ms": 3.699, "count": 125, "max_ms": 6.733, "p95_ms": 3.905}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 23.368, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2802}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4333.652, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55f946f8e680] left block unavailable for requested intra mode [h264 @ 0x55f946f8e680] error while decoding MB 0 38, bytestream 1951 [in#0/rtsp @ 0x55f9461d8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f9461d8e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 208, "h264_decode_error_count": 384, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:23:02+09:00", "last_error_summary": "[h264 @ 0x55f946f8e680] left block unavailable for requested intra mode [h264 @ 0x55f946f8e680] error while decoding MB 0 38, bytestream 1951 [in#0/rtsp @ 0x55f9461d8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f9461d8e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T04:18:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 302.328, "last_started_at": "2026-07-02T04:18:00+09:00", "pid": 37602, "process": {"exists": false, "pid": 37602}, "restart_count": 233, "restart_count_per_hour": 6, "rtsp_timeout_count": 100, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 234}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.712, "sample_count": 2569, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 336246, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:23:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.068, "last_frame_pushed_at": "2026-07-02T04:23:02+09:00", "last_frame_read_age_sec": 0.056, "last_frame_read_at": "2026-07-02T04:23:02+09:00", "mjpeg": {"age_sec": 0.053, "exists": true, "mtime": "2026-07-02T04:23:02.493878+09:00", "mtime_epoch": 1782933782.4938781, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 115285}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:23:02+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1023.094, "name": "celery", "pid": 237, "rss_mb": 1017.281, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 101, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.372, "pushed": 3.372, "read": 3.372}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.552, "count": 101, "max_ms": 2.197, "p95_ms": 1.8}, "gstreamer_push": {"avg_ms": 1.633, "count": 101, "max_ms": 6.322, "p95_ms": 3.921}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.851, "count": 101, "max_ms": 78.591, "p95_ms": 76.07}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.385, "count": 101, "max_ms": 79.815, "p95_ms": 78.802}, "yolo11_640_fire_11class": {"avg_ms": 75.72, "count": 101, "max_ms": 81.132, "p95_ms": 79.94}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 81.213, "count": 101, "max_ms": 87.552, "p95_ms": 86.024}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.622, "count": 101, "max_ms": 97.803, "p95_ms": 96.294}, "read_interval": {"avg_ms": 296.847, "count": 101, "max_ms": 19183.405, "p95_ms": 114.981}, "triton_wait": {"avg_ms": 81.305, "count": 101, "max_ms": 87.563, "p95_ms": 86.04}, "visualize": {"avg_ms": 3.638, "count": 101, "max_ms": 3.98, "p95_ms": 3.789}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.949, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2802}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4333.23, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T04:23:02+09:00", "last_error_summary": "[h264 @ 0x55f946f8e680] left block unavailable for requested intra mode [h264 @ 0x55f946f8e680] error while decoding MB 0 38, bytestream 1951 [in#0/rtsp @ 0x55f9461d8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f9461d8e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 208, "h264_decode_error_count": 384, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:23:02+09:00", "last_error_summary": "[h264 @ 0x55f946f8e680] left block unavailable for requested intra mode [h264 @ 0x55f946f8e680] error while decoding MB 0 38, bytestream 1951 [in#0/rtsp @ 0x55f9461d8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f9461d8e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T04:23:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 302.372, "last_started_at": "2026-07-02T04:18:00+09:00", "pid": 37602, "process": {"exists": false, "pid": 37602}, "restart_count": 233, "restart_count_per_hour": 6, "rtsp_timeout_count": 100, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 234}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.712, "sample_count": 2570, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 336247, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:23:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T04:23:02+09:00", "last_frame_read_age_sec": 0.099, "last_frame_read_at": "2026-07-02T04:23:02+09:00", "mjpeg": {"age_sec": 0.096, "exists": true, "mtime": "2026-07-02T04:23:02.493878+09:00", "mtime_epoch": 1782933782.4938781, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 115285}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:23:02+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1023.094, "name": "celery", "pid": 237, "rss_mb": 1017.281, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 101, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.401, "pushed": 3.401, "read": 3.368}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.551, "count": 102, "max_ms": 2.197, "p95_ms": 1.8}, "gstreamer_push": {"avg_ms": 1.63, "count": 102, "max_ms": 6.322, "p95_ms": 3.921}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.885, "count": 102, "max_ms": 78.591, "p95_ms": 76.07}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.444, "count": 102, "max_ms": 80.379, "p95_ms": 79.104}, "yolo11_640_fire_11class": {"avg_ms": 75.693, "count": 102, "max_ms": 81.132, "p95_ms": 79.94}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 81.212, "count": 102, "max_ms": 87.552, "p95_ms": 86.024}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.617, "count": 102, "max_ms": 97.803, "p95_ms": 96.294}, "read_interval": {"avg_ms": 296.847, "count": 101, "max_ms": 19183.405, "p95_ms": 114.981}, "triton_wait": {"avg_ms": 81.303, "count": 102, "max_ms": 87.563, "p95_ms": 86.04}, "visualize": {"avg_ms": 3.638, "count": 102, "max_ms": 3.98, "p95_ms": 3.789}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2802}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4333.016, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 208, "h264_decode_error_count": 384, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:23:02+09:00", "last_error_summary": "[h264 @ 0x55f946f8e680] left block unavailable for requested intra mode [h264 @ 0x55f946f8e680] error while decoding MB 0 38, bytestream 1951 [in#0/rtsp @ 0x55f9461d8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f9461d8e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T04:23:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 302.38, "last_started_at": "2026-07-02T04:18:00+09:00", "pid": 37602, "process": {"exists": false, "pid": 37602}, "restart_count": 233, "restart_count_per_hour": 6, "rtsp_timeout_count": 100, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 234}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.712, "sample_count": 2570, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 336247, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:23:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-02T04:23:02+09:00", "last_frame_read_age_sec": 0.107, "last_frame_read_at": "2026-07-02T04:23:02+09:00", "mjpeg": {"age_sec": 0.104, "exists": true, "mtime": "2026-07-02T04:23:02.493878+09:00", "mtime_epoch": 1782933782.4938781, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 115285}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:23:02+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1023.094, "name": "celery", "pid": 237, "rss_mb": 1017.281, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 102, "pushed_frames": 102, "read_frames": 101, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.4, "pushed": 3.4, "read": 3.367}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.551, "count": 101, "max_ms": 2.197, "p95_ms": 1.8}, "gstreamer_push": {"avg_ms": 1.63, "count": 102, "max_ms": 6.322, "p95_ms": 3.921}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.789, "count": 101, "max_ms": 78.591, "p95_ms": 76.07}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.478, "count": 101, "max_ms": 80.379, "p95_ms": 79.104}, "yolo11_640_fire_11class": {"avg_ms": 75.671, "count": 101, "max_ms": 81.132, "p95_ms": 79.94}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 81.24, "count": 101, "max_ms": 87.552, "p95_ms": 86.024}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.617, "count": 102, "max_ms": 97.803, "p95_ms": 96.294}, "read_interval": {"avg_ms": 296.847, "count": 101, "max_ms": 19183.405, "p95_ms": 114.981}, "triton_wait": {"avg_ms": 81.332, "count": 101, "max_ms": 87.563, "p95_ms": 86.04}, "visualize": {"avg_ms": 3.637, "count": 101, "max_ms": 3.98, "p95_ms": 3.789}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 2802}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4446.305, "limit_mb": 8796093022207.996}, "detail": {"pid": 37709, "restart_count": 234, "start_count": 235}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 208, "h264_decode_error_count": 384, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:23:02+09:00", "last_error_summary": "[h264 @ 0x55f946f8e680] left block unavailable for requested intra mode [h264 @ 0x55f946f8e680] error while decoding MB 0 38, bytestream 1951 [in#0/rtsp @ 0x55f9461d8e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f9461d8e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T04:23:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T04:23:05+09:00", "pid": 37709, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.477, "name": "ffmpeg", "pid": 37709, "rss_mb": 138.477, "state": "S (sleeping)", "threads": 106}, "restart_count": 234, "restart_count_per_hour": 7, "rtsp_timeout_count": 100, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 235}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.722, "sample_count": 2525, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 336247, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:23:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.784, "last_frame_pushed_at": "2026-07-02T04:23:02+09:00", "last_frame_read_age_sec": 4.878, "last_frame_read_at": "2026-07-02T04:23:02+09:00", "mjpeg": {"age_sec": 4.771, "exists": true, "mtime": "2026-07-02T04:23:02.597880+09:00", "mtime_epoch": 1782933782.5978796, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 115181}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:23:07+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1023.094, "name": "celery", "pid": 237, "rss_mb": 1017.281, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 58, "pushed_frames": 58, "read_frames": 57, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.936, "pushed": 1.936, "read": 1.903}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.567, "count": 58, "max_ms": 2.197, "p95_ms": 2.0}, "gstreamer_push": {"avg_ms": 1.794, "count": 58, "max_ms": 6.322, "p95_ms": 6.182}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.48, "count": 58, "max_ms": 76.574, "p95_ms": 74.596}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.609, "count": 58, "max_ms": 80.379, "p95_ms": 79.228}, "yolo11_640_fire_11class": {"avg_ms": 74.621, "count": 58, "max_ms": 79.756, "p95_ms": 78.815}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.99, "count": 58, "max_ms": 86.179, "p95_ms": 85.34}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.496, "count": 58, "max_ms": 96.733, "p95_ms": 95.857}, "read_interval": {"avg_ms": 441.628, "count": 57, "max_ms": 19183.405, "p95_ms": 119.106}, "triton_wait": {"avg_ms": 80.077, "count": 58, "max_ms": 86.191, "p95_ms": 85.352}, "visualize": {"avg_ms": 3.652, "count": 58, "max_ms": 3.98, "p95_ms": 3.884}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.956, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2840}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4360.109, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5622e54883c0] error while decoding MB 49 20, bytestream -5 [h264 @ 0x5622e552fec0] error while decoding MB 20 24, bytestream -9 [h264 @ 0x5622e5558240] error while decoding MB 54 41, bytestream -5 [in#0/rtsp @ 0x5622e5404e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5622e5404e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 209, "h264_decode_error_count": 387, "h264_decode_error_count_per_hour": 22, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:31:41+09:00", "last_error_summary": "[h264 @ 0x5622e54883c0] error while decoding MB 49 20, bytestream -5 [h264 @ 0x5622e552fec0] error while decoding MB 20 24, bytestream -9 [h264 @ 0x5622e5558240] error while decoding MB 54 41, bytestream -5 [in#0/rtsp @ 0x5622e5404e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5622e5404e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T04:23:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 516.591, "last_started_at": "2026-07-02T04:23:05+09:00", "pid": 37709, "process": {"exists": false, "pid": 37709}, "restart_count": 234, "restart_count_per_hour": 6, "rtsp_timeout_count": 102, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 235}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.183, "sample_count": 2589, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 340873, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:31:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.08, "last_frame_pushed_at": "2026-07-02T04:31:41+09:00", "last_frame_read_age_sec": 0.068, "last_frame_read_at": "2026-07-02T04:31:41+09:00", "mjpeg": {"age_sec": 0.064, "exists": true, "mtime": "2026-07-02T04:31:41.894380+09:00", "mtime_epoch": 1782934301.8943796, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 119476}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:31:41+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1030.262, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 98, "read_frames": 98, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.301, "pushed": 3.268, "read": 3.268}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.547, "count": 99, "max_ms": 2.095, "p95_ms": 1.669}, "gstreamer_push": {"avg_ms": 1.596, "count": 98, "max_ms": 6.068, "p95_ms": 3.885}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.992, "count": 99, "max_ms": 78.928, "p95_ms": 75.642}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.709, "count": 99, "max_ms": 87.159, "p95_ms": 82.807}, "yolo11_640_fire_11class": {"avg_ms": 72.238, "count": 99, "max_ms": 81.166, "p95_ms": 79.296}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.825, "count": 99, "max_ms": 88.005, "p95_ms": 86.74}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.047, "count": 99, "max_ms": 100.901, "p95_ms": 97.504}, "read_interval": {"avg_ms": 306.285, "count": 98, "max_ms": 10194.792, "p95_ms": 115.81}, "triton_wait": {"avg_ms": 78.602, "count": 99, "max_ms": 88.016, "p95_ms": 86.762}, "visualize": {"avg_ms": 3.682, "count": 99, "max_ms": 4.867, "p95_ms": 3.834}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2840}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4360.082, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T04:31:41+09:00", "last_error_summary": "[h264 @ 0x5622e54883c0] error while decoding MB 49 20, bytestream -5 [h264 @ 0x5622e552fec0] error while decoding MB 20 24, bytestream -9 [h264 @ 0x5622e5558240] error while decoding MB 54 41, bytestream -5 [in#0/rtsp @ 0x5622e5404e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5622e5404e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 209, "h264_decode_error_count": 387, "h264_decode_error_count_per_hour": 22, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:31:41+09:00", "last_error_summary": "[h264 @ 0x5622e54883c0] error while decoding MB 49 20, bytestream -5 [h264 @ 0x5622e552fec0] error while decoding MB 20 24, bytestream -9 [h264 @ 0x5622e5558240] error while decoding MB 54 41, bytestream -5 [in#0/rtsp @ 0x5622e5404e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5622e5404e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T04:31:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 516.607, "last_started_at": "2026-07-02T04:23:05+09:00", "pid": 37709, "process": {"exists": false, "pid": 37709}, "restart_count": 234, "restart_count_per_hour": 6, "rtsp_timeout_count": 102, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 235}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.183, "sample_count": 2589, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 340873, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:31:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-02T04:31:41+09:00", "last_frame_read_age_sec": 0.085, "last_frame_read_at": "2026-07-02T04:31:41+09:00", "mjpeg": {"age_sec": 0.081, "exists": true, "mtime": "2026-07-02T04:31:41.894380+09:00", "mtime_epoch": 1782934301.8943796, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 119476}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:31:41+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1030.262, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 98, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.268, "pushed": 3.268, "read": 3.268}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.548, "count": 98, "max_ms": 2.095, "p95_ms": 1.669}, "gstreamer_push": {"avg_ms": 1.636, "count": 98, "max_ms": 6.068, "p95_ms": 3.981}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.028, "count": 98, "max_ms": 78.928, "p95_ms": 75.642}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.671, "count": 98, "max_ms": 87.159, "p95_ms": 82.807}, "yolo11_640_fire_11class": {"avg_ms": 72.181, "count": 98, "max_ms": 81.166, "p95_ms": 79.296}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.763, "count": 98, "max_ms": 88.005, "p95_ms": 86.74}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.994, "count": 98, "max_ms": 100.901, "p95_ms": 97.504}, "read_interval": {"avg_ms": 306.285, "count": 98, "max_ms": 10194.792, "p95_ms": 115.81}, "triton_wait": {"avg_ms": 78.547, "count": 98, "max_ms": 88.016, "p95_ms": 86.762}, "visualize": {"avg_ms": 3.681, "count": 98, "max_ms": 4.867, "p95_ms": 3.834}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2840}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4359.891, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 209, "h264_decode_error_count": 387, "h264_decode_error_count_per_hour": 22, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:31:41+09:00", "last_error_summary": "[h264 @ 0x5622e54883c0] error while decoding MB 49 20, bytestream -5 [h264 @ 0x5622e552fec0] error while decoding MB 20 24, bytestream -9 [h264 @ 0x5622e5558240] error while decoding MB 54 41, bytestream -5 [in#0/rtsp @ 0x5622e5404e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5622e5404e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T04:31:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 516.616, "last_started_at": "2026-07-02T04:23:05+09:00", "pid": 37709, "process": {"exists": false, "pid": 37709}, "restart_count": 234, "restart_count_per_hour": 6, "rtsp_timeout_count": 102, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 235}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.183, "sample_count": 2588, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 340873, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:31:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.02, "last_frame_pushed_at": "2026-07-02T04:31:41+09:00", "last_frame_read_age_sec": 0.094, "last_frame_read_at": "2026-07-02T04:31:41+09:00", "mjpeg": {"age_sec": null, "exists": false, "mtime": null, "mtime_epoch": null, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": null}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:31:41+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1030.262, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 98, "pushed_frames": 98, "read_frames": 98, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.267, "pushed": 3.267, "read": 3.267}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.548, "count": 98, "max_ms": 2.095, "p95_ms": 1.669}, "gstreamer_push": {"avg_ms": 1.636, "count": 98, "max_ms": 6.068, "p95_ms": 3.981}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.028, "count": 98, "max_ms": 78.928, "p95_ms": 75.642}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.671, "count": 98, "max_ms": 87.159, "p95_ms": 82.807}, "yolo11_640_fire_11class": {"avg_ms": 72.181, "count": 98, "max_ms": 81.166, "p95_ms": 79.296}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.763, "count": 98, "max_ms": 88.005, "p95_ms": 86.74}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.994, "count": 98, "max_ms": 100.901, "p95_ms": 97.504}, "read_interval": {"avg_ms": 306.285, "count": 98, "max_ms": 10194.792, "p95_ms": 115.81}, "triton_wait": {"avg_ms": 78.547, "count": 98, "max_ms": 88.016, "p95_ms": 86.762}, "visualize": {"avg_ms": 3.681, "count": 98, "max_ms": 4.867, "p95_ms": 3.834}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2840}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4469.977, "limit_mb": 8796093022207.996}, "detail": {"pid": 37877, "restart_count": 235, "start_count": 236}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 209, "h264_decode_error_count": 387, "h264_decode_error_count_per_hour": 22, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:31:41+09:00", "last_error_summary": "[h264 @ 0x5622e54883c0] error while decoding MB 49 20, bytestream -5 [h264 @ 0x5622e552fec0] error while decoding MB 20 24, bytestream -9 [h264 @ 0x5622e5558240] error while decoding MB 54 41, bytestream -5 [in#0/rtsp @ 0x5622e5404e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5622e5404e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T04:31:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T04:31:44+09:00", "pid": 37877, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.883, "name": "ffmpeg", "pid": 37877, "rss_mb": 138.883, "state": "S (sleeping)", "threads": 106}, "restart_count": 235, "restart_count_per_hour": 7, "rtsp_timeout_count": 102, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 236}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.183, "sample_count": 2535, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 340873, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:31:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.755, "last_frame_pushed_at": "2026-07-02T04:31:41+09:00", "last_frame_read_age_sec": 4.829, "last_frame_read_at": "2026-07-02T04:31:41+09:00", "mjpeg": {"age_sec": 4.741, "exists": true, "mtime": "2026-07-02T04:31:41.978381+09:00", "mtime_epoch": 1782934301.9783812, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 120828}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:31:46+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1030.262, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 55, "pushed_frames": 55, "read_frames": 54, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.839, "pushed": 1.839, "read": 1.805}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.531, "count": 55, "max_ms": 1.922, "p95_ms": 1.574}, "gstreamer_push": {"avg_ms": 1.746, "count": 55, "max_ms": 6.068, "p95_ms": 5.877}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.411, "count": 55, "max_ms": 76.423, "p95_ms": 72.586}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.481, "count": 55, "max_ms": 87.159, "p95_ms": 84.163}, "yolo11_640_fire_11class": {"avg_ms": 68.142, "count": 55, "max_ms": 79.517, "p95_ms": 78.816}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.49, "count": 55, "max_ms": 87.813, "p95_ms": 86.751}}, "truncated_count": 0}, "processing_total": {"avg_ms": 85.397, "count": 55, "max_ms": 100.901, "p95_ms": 97.78}, "read_interval": {"avg_ms": 466.167, "count": 54, "max_ms": 10194.792, "p95_ms": 121.443}, "triton_wait": {"avg_ms": 74.994, "count": 55, "max_ms": 87.825, "p95_ms": 87.171}, "visualize": {"avg_ms": 3.659, "count": 55, "max_ms": 4.173, "p95_ms": 3.834}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.911, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3813}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4298.383, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55ff87db7940] error while decoding MB 96 28, bytestream -15 [in#0/rtsp @ 0x55ff87505e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ff87505e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 67, "h264_decode_error_count": 67, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:32:30+09:00", "last_error_summary": "[h264 @ 0x55ff87db7940] error while decoding MB 96 28, bytestream -15 [in#0/rtsp @ 0x55ff87505e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ff87505e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T04:12:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 1186.217, "last_started_at": "2026-07-02T04:12:44+09:00", "pid": 37450, "process": {"exists": false, "pid": 37450}, "restart_count": 66, "restart_count_per_hour": 8, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 67}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.41, "sample_count": 2604, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 457597, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:32:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.074, "last_frame_pushed_at": "2026-07-02T04:32:30+09:00", "last_frame_read_age_sec": 0.062, "last_frame_read_at": "2026-07-02T04:32:30+09:00", "mjpeg": {"age_sec": 0.057, "exists": true, "mtime": "2026-07-02T04:32:30.811212+09:00", "mtime_epoch": 1782934350.8112123, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97989}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T04:32:30+09:00", "process": {"exists": true, "fd_count": 137, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1085.301, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.746, "pushed": 0.688, "read": 0.746}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.609, "count": 13, "max_ms": 4.658, "p95_ms": 4.658}, "gstreamer_push": {"avg_ms": 1.601, "count": 12, "max_ms": 3.868, "p95_ms": 3.868}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.854, "count": 13, "max_ms": 57.103, "p95_ms": 57.103}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.082, "count": 13, "max_ms": 77.801, "p95_ms": 77.801}, "yolo11_640_fire_11class": {"avg_ms": 53.382, "count": 13, "max_ms": 57.682, "p95_ms": 57.682}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.279, "count": 13, "max_ms": 76.985, "p95_ms": 76.985}}, "truncated_count": 0}, "processing_total": {"avg_ms": 70.06, "count": 13, "max_ms": 89.9, "p95_ms": 89.9}, "read_interval": {"avg_ms": 2393.525, "count": 13, "max_ms": 16442.818, "p95_ms": 16442.818}, "triton_wait": {"avg_ms": 57.32, "count": 13, "max_ms": 77.813, "p95_ms": 77.813}, "visualize": {"avg_ms": 3.792, "count": 13, "max_ms": 4.686, "p95_ms": 4.686}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.438, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3813}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4298.312, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T04:32:30+09:00", "last_error_summary": "[h264 @ 0x55ff87db7940] error while decoding MB 96 28, bytestream -15 [in#0/rtsp @ 0x55ff87505e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ff87505e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 67, "h264_decode_error_count": 67, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:32:30+09:00", "last_error_summary": "[h264 @ 0x55ff87db7940] error while decoding MB 96 28, bytestream -15 [in#0/rtsp @ 0x55ff87505e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ff87505e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T04:32:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 1186.233, "last_started_at": "2026-07-02T04:12:44+09:00", "pid": 37450, "process": {"exists": false, "pid": 37450}, "restart_count": 66, "restart_count_per_hour": 8, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 67}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.41, "sample_count": 2604, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 457597, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:32:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-02T04:32:30+09:00", "last_frame_read_age_sec": 0.079, "last_frame_read_at": "2026-07-02T04:32:30+09:00", "mjpeg": {"age_sec": 0.073, "exists": true, "mtime": "2026-07-02T04:32:30.811212+09:00", "mtime_epoch": 1782934350.8112123, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 97989}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T04:32:30+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1085.301, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.745, "pushed": 0.745, "read": 0.745}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.609, "count": 13, "max_ms": 4.658, "p95_ms": 4.658}, "gstreamer_push": {"avg_ms": 1.578, "count": 13, "max_ms": 3.868, "p95_ms": 3.868}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.854, "count": 13, "max_ms": 57.103, "p95_ms": 57.103}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.082, "count": 13, "max_ms": 77.801, "p95_ms": 77.801}, "yolo11_640_fire_11class": {"avg_ms": 53.382, "count": 13, "max_ms": 57.682, "p95_ms": 57.682}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.279, "count": 13, "max_ms": 76.985, "p95_ms": 76.985}}, "truncated_count": 0}, "processing_total": {"avg_ms": 70.06, "count": 13, "max_ms": 89.9, "p95_ms": 89.9}, "read_interval": {"avg_ms": 2393.525, "count": 13, "max_ms": 16442.818, "p95_ms": 16442.818}, "triton_wait": {"avg_ms": 57.32, "count": 13, "max_ms": 77.813, "p95_ms": 77.813}, "visualize": {"avg_ms": 3.792, "count": 13, "max_ms": 4.686, "p95_ms": 4.686}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.454, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3813}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4298.273, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 67, "h264_decode_error_count": 67, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:32:30+09:00", "last_error_summary": "[h264 @ 0x55ff87db7940] error while decoding MB 96 28, bytestream -15 [in#0/rtsp @ 0x55ff87505e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ff87505e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T04:32:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 1186.241, "last_started_at": "2026-07-02T04:12:44+09:00", "pid": 37450, "process": {"exists": false, "pid": 37450}, "restart_count": 66, "restart_count_per_hour": 8, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 67}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.41, "sample_count": 2604, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 457597, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:32:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.019, "last_frame_pushed_at": "2026-07-02T04:32:30+09:00", "last_frame_read_age_sec": 0.086, "last_frame_read_at": "2026-07-02T04:32:30+09:00", "mjpeg": {"age_sec": 0.001, "exists": true, "mtime": "2026-07-02T04:32:30.891214+09:00", "mtime_epoch": 1782934350.8912137, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 98612}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T04:32:30+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1085.301, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.744, "pushed": 0.744, "read": 0.744}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.609, "count": 13, "max_ms": 4.658, "p95_ms": 4.658}, "gstreamer_push": {"avg_ms": 1.578, "count": 13, "max_ms": 3.868, "p95_ms": 3.868}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.854, "count": 13, "max_ms": 57.103, "p95_ms": 57.103}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.082, "count": 13, "max_ms": 77.801, "p95_ms": 77.801}, "yolo11_640_fire_11class": {"avg_ms": 53.382, "count": 13, "max_ms": 57.682, "p95_ms": 57.682}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.279, "count": 13, "max_ms": 76.985, "p95_ms": 76.985}}, "truncated_count": 0}, "processing_total": {"avg_ms": 70.06, "count": 13, "max_ms": 89.9, "p95_ms": 89.9}, "read_interval": {"avg_ms": 2393.525, "count": 13, "max_ms": 16442.818, "p95_ms": 16442.818}, "triton_wait": {"avg_ms": 57.32, "count": 13, "max_ms": 77.813, "p95_ms": 77.813}, "visualize": {"avg_ms": 3.792, "count": 13, "max_ms": 4.686, "p95_ms": 4.686}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.462, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 3813}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4318.621, "limit_mb": 8796093022207.996}, "detail": {"pid": 37984, "restart_count": 67, "start_count": 68}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 67, "h264_decode_error_count": 67, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:32:30+09:00", "last_error_summary": "[h264 @ 0x55ff87db7940] error while decoding MB 96 28, bytestream -15 [in#0/rtsp @ 0x55ff87505e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ff87505e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T04:32:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T04:32:33+09:00", "pid": 37984, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.008, "name": "ffmpeg", "pid": 37984, "rss_mb": 58.008, "state": "S (sleeping)", "threads": 1}, "restart_count": 67, "restart_count_per_hour": 9, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 68}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 98.41, "sample_count": 2561, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 457597, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:32:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.652, "last_frame_pushed_at": "2026-07-02T04:32:30+09:00", "last_frame_read_age_sec": 4.719, "last_frame_read_at": "2026-07-02T04:32:30+09:00", "mjpeg": {"age_sec": 4.634, "exists": true, "mtime": "2026-07-02T04:32:30.891214+09:00", "mtime_epoch": 1782934350.8912137, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 98612}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T04:32:35+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1161.414, "name": "celery", "pid": 267, "rss_mb": 1085.301, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.588, "pushed": 0.588, "read": 0.588}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.609, "count": 13, "max_ms": 4.658, "p95_ms": 4.658}, "gstreamer_push": {"avg_ms": 1.578, "count": 13, "max_ms": 3.868, "p95_ms": 3.868}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.854, "count": 13, "max_ms": 57.103, "p95_ms": 57.103}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.082, "count": 13, "max_ms": 77.801, "p95_ms": 77.801}, "yolo11_640_fire_11class": {"avg_ms": 53.382, "count": 13, "max_ms": 57.682, "p95_ms": 57.682}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.279, "count": 13, "max_ms": 76.985, "p95_ms": 76.985}}, "truncated_count": 0}, "processing_total": {"avg_ms": 70.06, "count": 13, "max_ms": 89.9, "p95_ms": 89.9}, "read_interval": {"avg_ms": 2393.525, "count": 13, "max_ms": 16442.818, "p95_ms": 16442.818}, "triton_wait": {"avg_ms": 57.32, "count": 13, "max_ms": 77.813, "p95_ms": 77.813}, "visualize": {"avg_ms": 3.792, "count": 13, "max_ms": 4.686, "p95_ms": 4.686}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.094, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2879}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4362.379, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x559d532cfec0] error while decoding MB 23 7, bytestream -7 [h264 @ 0x559d53d03340] left block unavailable for requested intra mode [h264 @ 0x559d53d03340] error while decoding MB 0 33, bytestream 1703 [in#0/rtsp @ 0x559d531b1e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559d531b1e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 210, "h264_decode_error_count": 389, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:40:29+09:00", "last_error_summary": "[h264 @ 0x559d532cfec0] error while decoding MB 23 7, bytestream -7 [h264 @ 0x559d53d03340] left block unavailable for requested intra mode [h264 @ 0x559d53d03340] error while decoding MB 0 33, bytestream 1703 [in#0/rtsp @ 0x559d531b1e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559d531b1e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T04:31:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 524.809, "last_started_at": "2026-07-02T04:31:44+09:00", "pid": 37877, "process": {"exists": false, "pid": 37877}, "restart_count": 235, "restart_count_per_hour": 7, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 236}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.439, "sample_count": 2589, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 345521, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:40:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.078, "last_frame_pushed_at": "2026-07-02T04:40:29+09:00", "last_frame_read_age_sec": 0.068, "last_frame_read_at": "2026-07-02T04:40:29+09:00", "mjpeg": {"age_sec": 0.064, "exists": true, "mtime": "2026-07-02T04:40:29.461984+09:00", "mtime_epoch": 1782934829.4619844, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 144620}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:40:29+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1033.738, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 86, "pushed_frames": 86, "read_frames": 86, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.868, "pushed": 2.868, "read": 2.868}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.025, "count": 85, "max_ms": 6.142, "p95_ms": 4.401}, "gstreamer_push": {"avg_ms": 1.328, "count": 86, "max_ms": 3.134, "p95_ms": 1.49}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.674, "count": 85, "max_ms": 72.151, "p95_ms": 67.19}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.968, "count": 85, "max_ms": 91.975, "p95_ms": 88.066}, "yolo11_640_fire_11class": {"avg_ms": 62.389, "count": 85, "max_ms": 84.132, "p95_ms": 79.507}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.855, "count": 85, "max_ms": 91.534, "p95_ms": 88.959}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.808, "count": 86, "max_ms": 145.22, "p95_ms": 141.23}, "read_interval": {"avg_ms": 349.407, "count": 86, "max_ms": 10160.645, "p95_ms": 154.586}, "triton_wait": {"avg_ms": 71.022, "count": 85, "max_ms": 91.987, "p95_ms": 89.362}, "visualize": {"avg_ms": 4.54, "count": 85, "max_ms": 5.929, "p95_ms": 5.484}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.986, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2879}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4361.902, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T04:40:29+09:00", "last_error_summary": "[h264 @ 0x559d532cfec0] error while decoding MB 23 7, bytestream -7 [h264 @ 0x559d53d03340] left block unavailable for requested intra mode [h264 @ 0x559d53d03340] error while decoding MB 0 33, bytestream 1703 [in#0/rtsp @ 0x559d531b1e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559d531b1e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 210, "h264_decode_error_count": 389, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:40:29+09:00", "last_error_summary": "[h264 @ 0x559d532cfec0] error while decoding MB 23 7, bytestream -7 [h264 @ 0x559d53d03340] left block unavailable for requested intra mode [h264 @ 0x559d53d03340] error while decoding MB 0 33, bytestream 1703 [in#0/rtsp @ 0x559d531b1e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559d531b1e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T04:40:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 524.871, "last_started_at": "2026-07-02T04:31:44+09:00", "pid": 37877, "process": {"exists": false, "pid": 37877}, "restart_count": 235, "restart_count_per_hour": 7, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 236}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.439, "sample_count": 2589, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 345522, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:40:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T04:40:29+09:00", "last_frame_read_age_sec": 0.13, "last_frame_read_at": "2026-07-02T04:40:29+09:00", "mjpeg": {"age_sec": 0.127, "exists": true, "mtime": "2026-07-02T04:40:29.461984+09:00", "mtime_epoch": 1782934829.4619844, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 144620}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:40:29+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1033.738, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 86, "pushed_frames": 86, "read_frames": 85, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.874, "pushed": 2.874, "read": 2.84}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.042, "count": 86, "max_ms": 6.142, "p95_ms": 4.473}, "gstreamer_push": {"avg_ms": 1.328, "count": 86, "max_ms": 3.134, "p95_ms": 1.49}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.649, "count": 86, "max_ms": 72.151, "p95_ms": 67.19}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.882, "count": 86, "max_ms": 91.975, "p95_ms": 88.066}, "yolo11_640_fire_11class": {"avg_ms": 62.286, "count": 86, "max_ms": 84.132, "p95_ms": 79.507}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.774, "count": 86, "max_ms": 91.534, "p95_ms": 88.959}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.712, "count": 86, "max_ms": 145.22, "p95_ms": 141.23}, "read_interval": {"avg_ms": 351.817, "count": 85, "max_ms": 10160.645, "p95_ms": 154.586}, "triton_wait": {"avg_ms": 70.924, "count": 86, "max_ms": 91.987, "p95_ms": 89.362}, "visualize": {"avg_ms": 4.605, "count": 86, "max_ms": 10.135, "p95_ms": 5.61}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.925, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2879}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4361.895, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 210, "h264_decode_error_count": 389, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:40:29+09:00", "last_error_summary": "[h264 @ 0x559d532cfec0] error while decoding MB 23 7, bytestream -7 [h264 @ 0x559d53d03340] left block unavailable for requested intra mode [h264 @ 0x559d53d03340] error while decoding MB 0 33, bytestream 1703 [in#0/rtsp @ 0x559d531b1e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559d531b1e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T04:40:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 524.878, "last_started_at": "2026-07-02T04:31:44+09:00", "pid": 37877, "process": {"exists": false, "pid": 37877}, "restart_count": 235, "restart_count_per_hour": 7, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 236}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.439, "sample_count": 2589, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 345522, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:40:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-02T04:40:29+09:00", "last_frame_read_age_sec": 0.138, "last_frame_read_at": "2026-07-02T04:40:29+09:00", "mjpeg": {"age_sec": 0.134, "exists": true, "mtime": "2026-07-02T04:40:29.461984+09:00", "mtime_epoch": 1782934829.4619844, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 144620}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:40:29+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1033.738, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 86, "pushed_frames": 86, "read_frames": 85, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.873, "pushed": 2.873, "read": 2.84}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.042, "count": 86, "max_ms": 6.142, "p95_ms": 4.473}, "gstreamer_push": {"avg_ms": 1.328, "count": 86, "max_ms": 3.134, "p95_ms": 1.49}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.649, "count": 86, "max_ms": 72.151, "p95_ms": 67.19}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.882, "count": 86, "max_ms": 91.975, "p95_ms": 88.066}, "yolo11_640_fire_11class": {"avg_ms": 62.286, "count": 86, "max_ms": 84.132, "p95_ms": 79.507}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.774, "count": 86, "max_ms": 91.534, "p95_ms": 88.959}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.712, "count": 86, "max_ms": 145.22, "p95_ms": 141.23}, "read_interval": {"avg_ms": 351.817, "count": 85, "max_ms": 10160.645, "p95_ms": 154.586}, "triton_wait": {"avg_ms": 70.924, "count": 86, "max_ms": 91.987, "p95_ms": 89.362}, "visualize": {"avg_ms": 4.605, "count": 86, "max_ms": 10.135, "p95_ms": 5.61}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.932, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2879}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4472.328, "limit_mb": 8796093022207.996}, "detail": {"pid": 38136, "restart_count": 236, "start_count": 237}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 210, "h264_decode_error_count": 389, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:40:29+09:00", "last_error_summary": "[h264 @ 0x559d532cfec0] error while decoding MB 23 7, bytestream -7 [h264 @ 0x559d53d03340] left block unavailable for requested intra mode [h264 @ 0x559d53d03340] error while decoding MB 0 33, bytestream 1703 [in#0/rtsp @ 0x559d531b1e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559d531b1e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T04:40:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T04:40:31+09:00", "pid": 38136, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.211, "name": "ffmpeg", "pid": 38136, "rss_mb": 139.211, "state": "S (sleeping)", "threads": 106}, "restart_count": 236, "restart_count_per_hour": 8, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 237}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.439, "sample_count": 2549, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 345522, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:40:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.28, "last_frame_pushed_at": "2026-07-02T04:40:29+09:00", "last_frame_read_age_sec": 4.406, "last_frame_read_at": "2026-07-02T04:40:29+09:00", "mjpeg": {"age_sec": 4.266, "exists": true, "mtime": "2026-07-02T04:40:29.597986+09:00", "mtime_epoch": 1782934829.5979865, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 143930}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:40:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1033.738, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 55, "pushed_frames": 55, "read_frames": 54, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.839, "pushed": 1.839, "read": 1.806}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 2.367, "count": 55, "max_ms": 4.473, "p95_ms": 4.292}, "gstreamer_push": {"avg_ms": 1.327, "count": 55, "max_ms": 2.026, "p95_ms": 1.448}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.283, "count": 55, "max_ms": 72.151, "p95_ms": 65.942}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.859, "count": 55, "max_ms": 90.522, "p95_ms": 87.239}, "yolo11_640_fire_11class": {"avg_ms": 63.059, "count": 55, "max_ms": 84.132, "p95_ms": 80.317}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.943, "count": 55, "max_ms": 91.534, "p95_ms": 88.959}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.697, "count": 55, "max_ms": 145.22, "p95_ms": 141.23}, "read_interval": {"avg_ms": 474.757, "count": 54, "max_ms": 10160.645, "p95_ms": 159.96}, "triton_wait": {"avg_ms": 70.206, "count": 55, "max_ms": 91.547, "p95_ms": 89.362}, "visualize": {"avg_ms": 4.288, "count": 55, "max_ms": 10.135, "p95_ms": 5.61}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.904, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2905}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4366.074, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55c6428b6140] left block unavailable for requested intra mode [h264 @ 0x55c6428b6140] error while decoding MB 0 17, bytestream 34126 [h264 @ 0x55c6429df480] cabac decode of qscale diff failed at 66 4 [h264 @ 0x55c6429df480] error while decoding MB 66 4, bytestream 981 [h264 @ 0x55c6429cb2c0] error while decoding MB 55 3, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 211, "h264_decode_error_count": 393, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:46:43+09:00", "last_error_summary": "[h264 @ 0x55c6428b6140] left block unavailable for requested intra mode [h264 @ 0x55c6428b6140] error while decoding MB 0 17, bytestream 34126 [h264 @ 0x55c6429df480] cabac decode of qscale diff failed at 66 4 [h264 @ 0x55c6429df480] error while decoding MB 66 4, bytestream 981 [h264 @ 0x55c6429cb2c0] error while decoding MB 55 3, bytestream -5", "last_exit_at": "2026-07-02T04:40:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 371.441, "last_started_at": "2026-07-02T04:40:31+09:00", "pid": 38136, "process": {"exists": false, "pid": 38136}, "restart_count": 236, "restart_count_per_hour": 7, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 237}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.731, "sample_count": 2456, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 348617, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:46:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.076, "last_frame_pushed_at": "2026-07-02T04:46:43+09:00", "last_frame_read_age_sec": 0.066, "last_frame_read_at": "2026-07-02T04:46:43+09:00", "mjpeg": {"age_sec": 0.057, "exists": true, "mtime": "2026-07-02T04:46:43.247599+09:00", "mtime_epoch": 1782935203.2475991, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 130486}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:46:43+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1032.352, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 229, "pushed_frames": 229, "read_frames": 229, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.651, "pushed": 7.651, "read": 7.651}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.641, "count": 229, "max_ms": 7.139, "p95_ms": 5.454}, "gstreamer_push": {"avg_ms": 1.291, "count": 229, "max_ms": 6.143, "p95_ms": 1.659}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.885, "count": 229, "max_ms": 76.3, "p95_ms": 67.085}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.162, "count": 229, "max_ms": 92.826, "p95_ms": 90.6}, "yolo11_640_fire_11class": {"avg_ms": 56.928, "count": 229, "max_ms": 79.354, "p95_ms": 74.781}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.02, "count": 229, "max_ms": 94.098, "p95_ms": 91.478}}, "truncated_count": 0}, "processing_total": {"avg_ms": 116.153, "count": 229, "max_ms": 184.032, "p95_ms": 158.784}, "read_interval": {"avg_ms": 131.071, "count": 229, "max_ms": 299.803, "p95_ms": 173.424}, "triton_wait": {"avg_ms": 69.49, "count": 229, "max_ms": 94.11, "p95_ms": 91.582}, "visualize": {"avg_ms": 5.139, "count": 229, "max_ms": 9.26, "p95_ms": 6.614}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.93, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2905}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4365.098, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T04:46:43+09:00", "last_error_summary": "[h264 @ 0x55c6428b6140] left block unavailable for requested intra mode [h264 @ 0x55c6428b6140] error while decoding MB 0 17, bytestream 34126 [h264 @ 0x55c6429df480] cabac decode of qscale diff failed at 66 4 [h264 @ 0x55c6429df480] error while decoding MB 66 4, bytestream 981 [h264 @ 0x55c6429cb2c0] error while decoding MB 55 3, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 211, "h264_decode_error_count": 393, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:46:43+09:00", "last_error_summary": "[h264 @ 0x55c6428b6140] left block unavailable for requested intra mode [h264 @ 0x55c6428b6140] error while decoding MB 0 17, bytestream 34126 [h264 @ 0x55c6429df480] cabac decode of qscale diff failed at 66 4 [h264 @ 0x55c6429df480] error while decoding MB 66 4, bytestream 981 [h264 @ 0x55c6429cb2c0] error while decoding MB 55 3, bytestream -5", "last_exit_at": "2026-07-02T04:46:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 371.538, "last_started_at": "2026-07-02T04:40:31+09:00", "pid": 38136, "process": {"exists": false, "pid": 38136}, "restart_count": 236, "restart_count_per_hour": 7, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 237}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.837, "sample_count": 2456, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 348618, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:46:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T04:46:43+09:00", "last_frame_read_age_sec": 0.162, "last_frame_read_at": "2026-07-02T04:46:43+09:00", "mjpeg": {"age_sec": 0.153, "exists": true, "mtime": "2026-07-02T04:46:43.247599+09:00", "mtime_epoch": 1782935203.2475991, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 130486}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:46:43+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1032.352, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 229, "pushed_frames": 229, "read_frames": 228, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.666, "pushed": 7.666, "read": 7.632}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.642, "count": 229, "max_ms": 7.139, "p95_ms": 5.454}, "gstreamer_push": {"avg_ms": 1.292, "count": 229, "max_ms": 6.143, "p95_ms": 1.659}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.923, "count": 229, "max_ms": 76.3, "p95_ms": 67.085}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.14, "count": 229, "max_ms": 92.826, "p95_ms": 90.6}, "yolo11_640_fire_11class": {"avg_ms": 56.894, "count": 229, "max_ms": 79.354, "p95_ms": 74.781}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.992, "count": 229, "max_ms": 94.098, "p95_ms": 91.478}}, "truncated_count": 0}, "processing_total": {"avg_ms": 116.172, "count": 229, "max_ms": 184.032, "p95_ms": 158.784}, "read_interval": {"avg_ms": 130.933, "count": 228, "max_ms": 299.803, "p95_ms": 173.424}, "triton_wait": {"avg_ms": 69.468, "count": 229, "max_ms": 94.11, "p95_ms": 91.582}, "visualize": {"avg_ms": 5.15, "count": 229, "max_ms": 9.26, "p95_ms": 6.658}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.874, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2905}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4365.09, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 211, "h264_decode_error_count": 393, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:46:43+09:00", "last_error_summary": "[h264 @ 0x55c6428b6140] left block unavailable for requested intra mode [h264 @ 0x55c6428b6140] error while decoding MB 0 17, bytestream 34126 [h264 @ 0x55c6429df480] cabac decode of qscale diff failed at 66 4 [h264 @ 0x55c6429df480] error while decoding MB 66 4, bytestream 981 [h264 @ 0x55c6429cb2c0] error while decoding MB 55 3, bytestream -5", "last_exit_at": "2026-07-02T04:46:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 371.542, "last_started_at": "2026-07-02T04:40:31+09:00", "pid": 38136, "process": {"exists": false, "pid": 38136}, "restart_count": 236, "restart_count_per_hour": 7, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 237}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.837, "sample_count": 2456, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 348618, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:46:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-02T04:46:43+09:00", "last_frame_read_age_sec": 0.167, "last_frame_read_at": "2026-07-02T04:46:43+09:00", "mjpeg": {"age_sec": 0.158, "exists": true, "mtime": "2026-07-02T04:46:43.247599+09:00", "mtime_epoch": 1782935203.2475991, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 130486}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:46:43+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1032.352, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 229, "pushed_frames": 229, "read_frames": 228, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.664, "pushed": 7.664, "read": 7.631}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.642, "count": 229, "max_ms": 7.139, "p95_ms": 5.454}, "gstreamer_push": {"avg_ms": 1.292, "count": 229, "max_ms": 6.143, "p95_ms": 1.659}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.923, "count": 229, "max_ms": 76.3, "p95_ms": 67.085}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.14, "count": 229, "max_ms": 92.826, "p95_ms": 90.6}, "yolo11_640_fire_11class": {"avg_ms": 56.894, "count": 229, "max_ms": 79.354, "p95_ms": 74.781}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.992, "count": 229, "max_ms": 94.098, "p95_ms": 91.478}}, "truncated_count": 0}, "processing_total": {"avg_ms": 116.172, "count": 229, "max_ms": 184.032, "p95_ms": 158.784}, "read_interval": {"avg_ms": 130.933, "count": 228, "max_ms": 299.803, "p95_ms": 173.424}, "triton_wait": {"avg_ms": 69.468, "count": 229, "max_ms": 94.11, "p95_ms": 91.582}, "visualize": {"avg_ms": 5.15, "count": 229, "max_ms": 9.26, "p95_ms": 6.658}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.878, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2905}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4473.406, "limit_mb": 8796093022207.996}, "detail": {"pid": 38243, "restart_count": 237, "start_count": 238}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 211, "h264_decode_error_count": 393, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:46:43+09:00", "last_error_summary": "[h264 @ 0x55c6428b6140] left block unavailable for requested intra mode [h264 @ 0x55c6428b6140] error while decoding MB 0 17, bytestream 34126 [h264 @ 0x55c6429df480] cabac decode of qscale diff failed at 66 4 [h264 @ 0x55c6429df480] error while decoding MB 66 4, bytestream 981 [h264 @ 0x55c6429cb2c0] error while decoding MB 55 3, bytestream -5", "last_exit_at": "2026-07-02T04:46:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T04:46:45+09:00", "pid": 38243, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.031, "name": "ffmpeg", "pid": 38243, "rss_mb": 139.031, "state": "S (sleeping)", "threads": 106}, "restart_count": 237, "restart_count_per_hour": 8, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 238}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.956, "sample_count": 2413, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 348618, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:46:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.528, "last_frame_pushed_at": "2026-07-02T04:46:43+09:00", "last_frame_read_age_sec": 4.686, "last_frame_read_at": "2026-07-02T04:46:43+09:00", "mjpeg": {"age_sec": 4.509, "exists": true, "mtime": "2026-07-02T04:46:43.415602+09:00", "mtime_epoch": 1782935203.4156017, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 132201}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:46:47+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1032.352, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 198, "pushed_frames": 198, "read_frames": 197, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.617, "pushed": 6.617, "read": 6.584}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 3.535, "count": 198, "max_ms": 7.139, "p95_ms": 5.29}, "gstreamer_push": {"avg_ms": 1.309, "count": 198, "max_ms": 6.143, "p95_ms": 1.796}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.35, "count": 198, "max_ms": 76.3, "p95_ms": 67.567}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.03, "count": 198, "max_ms": 92.826, "p95_ms": 90.986}, "yolo11_640_fire_11class": {"avg_ms": 56.859, "count": 198, "max_ms": 79.354, "p95_ms": 74.781}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.998, "count": 198, "max_ms": 94.098, "p95_ms": 92.075}}, "truncated_count": 0}, "processing_total": {"avg_ms": 114.216, "count": 198, "max_ms": 184.032, "p95_ms": 158.636}, "read_interval": {"avg_ms": 128.78, "count": 197, "max_ms": 299.803, "p95_ms": 170.541}, "triton_wait": {"avg_ms": 69.377, "count": 198, "max_ms": 94.11, "p95_ms": 92.162}, "visualize": {"avg_ms": 5.047, "count": 198, "max_ms": 8.523, "p95_ms": 6.579}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.922, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2914}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4370.133, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55e6cf93a880] error while decoding MB 47 8, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 212, "h264_decode_error_count": 394, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:49:29+09:00", "last_error_summary": "[h264 @ 0x55e6cf93a880] error while decoding MB 47 8, bytestream -7", "last_exit_at": "2026-07-02T04:46:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.649, "last_started_at": "2026-07-02T04:46:45+09:00", "pid": 38243, "process": {"exists": false, "pid": 38243}, "restart_count": 237, "restart_count_per_hour": 8, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 238}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 168.03, "sample_count": 2090, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 349702, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:49:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-02T04:49:29+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-02T04:49:29+09:00", "mjpeg": {"age_sec": 0.038, "exists": true, "mtime": "2026-07-02T04:49:29.534225+09:00", "mtime_epoch": 1782935369.5342247, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 126916}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:49:29+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1038.293, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 188, "pushed_frames": 188, "read_frames": 188, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.29, "pushed": 6.29, "read": 6.29}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.493, "count": 188, "max_ms": 7.517, "p95_ms": 6.738}, "gstreamer_push": {"avg_ms": 1.113, "count": 188, "max_ms": 4.052, "p95_ms": 1.516}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.614, "count": 188, "max_ms": 76.796, "p95_ms": 66.067}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.338, "count": 188, "max_ms": 96.679, "p95_ms": 86.959}, "yolo11_640_fire_11class": {"avg_ms": 53.818, "count": 188, "max_ms": 76.809, "p95_ms": 69.464}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.88, "count": 188, "max_ms": 95.318, "p95_ms": 87.726}}, "truncated_count": 0}, "processing_total": {"avg_ms": 147.885, "count": 188, "max_ms": 214.15, "p95_ms": 192.722}, "read_interval": {"avg_ms": 159.565, "count": 188, "max_ms": 224.278, "p95_ms": 202.83}, "triton_wait": {"avg_ms": 67.696, "count": 188, "max_ms": 96.691, "p95_ms": 87.905}, "visualize": {"avg_ms": 6.549, "count": 188, "max_ms": 9.247, "p95_ms": 8.446}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.888, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2914}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4369.555, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T04:49:29+09:00", "last_error_summary": "[h264 @ 0x55e6cf93a880] error while decoding MB 47 8, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 212, "h264_decode_error_count": 394, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:49:29+09:00", "last_error_summary": "[h264 @ 0x55e6cf93a880] error while decoding MB 47 8, bytestream -7", "last_exit_at": "2026-07-02T04:49:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.745, "last_started_at": "2026-07-02T04:46:45+09:00", "pid": 38243, "process": {"exists": false, "pid": 38243}, "restart_count": 237, "restart_count_per_hour": 8, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 238}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 168.03, "sample_count": 2090, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 349703, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:49:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T04:49:29+09:00", "last_frame_read_age_sec": 0.148, "last_frame_read_at": "2026-07-02T04:49:29+09:00", "mjpeg": {"age_sec": 0.134, "exists": true, "mtime": "2026-07-02T04:49:29.534225+09:00", "mtime_epoch": 1782935369.5342247, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 126916}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:49:29+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1038.293, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 189, "pushed_frames": 189, "read_frames": 188, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.303, "pushed": 6.303, "read": 6.27}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.485, "count": 188, "max_ms": 7.517, "p95_ms": 6.738}, "gstreamer_push": {"avg_ms": 1.113, "count": 189, "max_ms": 4.052, "p95_ms": 1.516}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.61, "count": 188, "max_ms": 76.796, "p95_ms": 66.067}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.341, "count": 188, "max_ms": 96.679, "p95_ms": 86.959}, "yolo11_640_fire_11class": {"avg_ms": 53.856, "count": 188, "max_ms": 76.809, "p95_ms": 69.464}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.881, "count": 188, "max_ms": 95.318, "p95_ms": 87.726}}, "truncated_count": 0}, "processing_total": {"avg_ms": 147.851, "count": 189, "max_ms": 214.15, "p95_ms": 192.722}, "read_interval": {"avg_ms": 159.565, "count": 188, "max_ms": 224.278, "p95_ms": 202.83}, "triton_wait": {"avg_ms": 67.698, "count": 188, "max_ms": 96.691, "p95_ms": 87.905}, "visualize": {"avg_ms": 6.542, "count": 188, "max_ms": 9.247, "p95_ms": 8.446}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.984, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2914}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4369.547, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 212, "h264_decode_error_count": 394, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:49:29+09:00", "last_error_summary": "[h264 @ 0x55e6cf93a880] error while decoding MB 47 8, bytestream -7", "last_exit_at": "2026-07-02T04:49:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.75, "last_started_at": "2026-07-02T04:46:45+09:00", "pid": 38243, "process": {"exists": false, "pid": 38243}, "restart_count": 237, "restart_count_per_hour": 8, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 238}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 168.03, "sample_count": 2090, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 349703, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:49:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-02T04:49:29+09:00", "last_frame_read_age_sec": 0.152, "last_frame_read_at": "2026-07-02T04:49:29+09:00", "mjpeg": {"age_sec": 0.138, "exists": true, "mtime": "2026-07-02T04:49:29.534225+09:00", "mtime_epoch": 1782935369.5342247, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 126916}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:49:29+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1038.293, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 189, "pushed_frames": 189, "read_frames": 188, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.302, "pushed": 6.302, "read": 6.269}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.485, "count": 188, "max_ms": 7.517, "p95_ms": 6.738}, "gstreamer_push": {"avg_ms": 1.113, "count": 189, "max_ms": 4.052, "p95_ms": 1.516}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.61, "count": 188, "max_ms": 76.796, "p95_ms": 66.067}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.341, "count": 188, "max_ms": 96.679, "p95_ms": 86.959}, "yolo11_640_fire_11class": {"avg_ms": 53.856, "count": 188, "max_ms": 76.809, "p95_ms": 69.464}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.881, "count": 188, "max_ms": 95.318, "p95_ms": 87.726}}, "truncated_count": 0}, "processing_total": {"avg_ms": 147.851, "count": 189, "max_ms": 214.15, "p95_ms": 192.722}, "read_interval": {"avg_ms": 159.565, "count": 188, "max_ms": 224.278, "p95_ms": 202.83}, "triton_wait": {"avg_ms": 67.698, "count": 188, "max_ms": 96.691, "p95_ms": 87.905}, "visualize": {"avg_ms": 6.542, "count": 188, "max_ms": 9.247, "p95_ms": 8.446}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.988, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2914}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4480.605, "limit_mb": 8796093022207.996}, "detail": {"pid": 38350, "restart_count": 238, "start_count": 239}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 212, "h264_decode_error_count": 394, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:49:29+09:00", "last_error_summary": "[h264 @ 0x55e6cf93a880] error while decoding MB 47 8, bytestream -7", "last_exit_at": "2026-07-02T04:49:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T04:49:31+09:00", "pid": 38350, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.719, "name": "ffmpeg", "pid": 38350, "rss_mb": 138.719, "state": "S (sleeping)", "threads": 106}, "restart_count": 238, "restart_count_per_hour": 9, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 239}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 168.113, "sample_count": 2060, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 349703, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:49:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.03, "last_frame_pushed_at": "2026-07-02T04:49:29+09:00", "last_frame_read_age_sec": 4.174, "last_frame_read_at": "2026-07-02T04:49:29+09:00", "mjpeg": {"age_sec": 4.012, "exists": true, "mtime": "2026-07-02T04:49:29.682227+09:00", "mtime_epoch": 1782935369.6822271, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 127295}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:49:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1038.293, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 164, "pushed_frames": 164, "read_frames": 163, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.487, "pushed": 5.487, "read": 5.453}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.457, "count": 164, "max_ms": 7.517, "p95_ms": 6.774}, "gstreamer_push": {"avg_ms": 1.13, "count": 164, "max_ms": 4.052, "p95_ms": 1.531}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.582, "count": 164, "max_ms": 76.796, "p95_ms": 66.067}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.252, "count": 164, "max_ms": 96.679, "p95_ms": 85.779}, "yolo11_640_fire_11class": {"avg_ms": 53.762, "count": 164, "max_ms": 76.809, "p95_ms": 69.151}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.727, "count": 164, "max_ms": 95.318, "p95_ms": 87.216}}, "truncated_count": 0}, "processing_total": {"avg_ms": 146.845, "count": 164, "max_ms": 208.279, "p95_ms": 192.722}, "read_interval": {"avg_ms": 158.627, "count": 163, "max_ms": 218.63, "p95_ms": 202.45}, "triton_wait": {"avg_ms": 67.603, "count": 164, "max_ms": 96.691, "p95_ms": 87.738}, "visualize": {"avg_ms": 6.509, "count": 164, "max_ms": 9.247, "p95_ms": 8.474}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.89, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2919}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4377.23, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5576dbd98d00] error while decoding MB 29 29, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 213, "h264_decode_error_count": 395, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:51:41+09:00", "last_error_summary": "[h264 @ 0x5576dbd98d00] error while decoding MB 29 29, bytestream -5", "last_exit_at": "2026-07-02T04:49:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.506, "last_started_at": "2026-07-02T04:49:31+09:00", "pid": 38350, "process": {"exists": false, "pid": 38350}, "restart_count": 238, "restart_count_per_hour": 8, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 239}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 201.873, "sample_count": 1797, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 350391, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:51:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-02T04:51:41+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-02T04:51:41+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-02T04:51:41.156338+09:00", "mtime_epoch": 1782935501.1563385, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 123778}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:51:41+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1040.816, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 152, "pushed_frames": 152, "read_frames": 152, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.076, "pushed": 5.076, "read": 5.076}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.804, "count": 151, "max_ms": 8.393, "p95_ms": 7.42}, "gstreamer_push": {"avg_ms": 0.933, "count": 152, "max_ms": 2.903, "p95_ms": 1.142}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.03, "count": 151, "max_ms": 71.756, "p95_ms": 67.492}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.67, "count": 151, "max_ms": 92.742, "p95_ms": 89.836}, "yolo11_640_fire_11class": {"avg_ms": 54.777, "count": 151, "max_ms": 72.927, "p95_ms": 70.549}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.137, "count": 151, "max_ms": 90.688, "p95_ms": 87.333}}, "truncated_count": 0}, "processing_total": {"avg_ms": 187.666, "count": 152, "max_ms": 249.196, "p95_ms": 216.181}, "read_interval": {"avg_ms": 197.831, "count": 152, "max_ms": 260.665, "p95_ms": 227.485}, "triton_wait": {"avg_ms": 68.974, "count": 151, "max_ms": 92.754, "p95_ms": 89.85}, "visualize": {"avg_ms": 8.521, "count": 151, "max_ms": 10.369, "p95_ms": 9.703}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.943, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2919}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4376.055, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T04:51:41+09:00", "last_error_summary": "[h264 @ 0x5576dbd98d00] error while decoding MB 29 29, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 213, "h264_decode_error_count": 395, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:51:41+09:00", "last_error_summary": "[h264 @ 0x5576dbd98d00] error while decoding MB 29 29, bytestream -5", "last_exit_at": "2026-07-02T04:51:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.639, "last_started_at": "2026-07-02T04:49:31+09:00", "pid": 38350, "process": {"exists": false, "pid": 38350}, "restart_count": 238, "restart_count_per_hour": 8, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 239}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 201.873, "sample_count": 1797, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 350392, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:51:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T04:51:41+09:00", "last_frame_read_age_sec": 0.183, "last_frame_read_at": "2026-07-02T04:51:41+09:00", "mjpeg": {"age_sec": 0.176, "exists": true, "mtime": "2026-07-02T04:51:41.156338+09:00", "mtime_epoch": 1782935501.1563385, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 123778}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:51:41+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1040.816, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 152, "pushed_frames": 152, "read_frames": 151, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.086, "pushed": 5.086, "read": 5.052}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.811, "count": 152, "max_ms": 8.393, "p95_ms": 7.423}, "gstreamer_push": {"avg_ms": 0.933, "count": 152, "max_ms": 2.903, "p95_ms": 1.142}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.969, "count": 152, "max_ms": 71.756, "p95_ms": 67.492}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.606, "count": 152, "max_ms": 92.742, "p95_ms": 89.836}, "yolo11_640_fire_11class": {"avg_ms": 54.736, "count": 152, "max_ms": 72.927, "p95_ms": 70.549}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.075, "count": 152, "max_ms": 90.688, "p95_ms": 87.333}}, "truncated_count": 0}, "processing_total": {"avg_ms": 187.671, "count": 152, "max_ms": 249.196, "p95_ms": 216.181}, "read_interval": {"avg_ms": 197.901, "count": 151, "max_ms": 260.665, "p95_ms": 227.485}, "triton_wait": {"avg_ms": 68.908, "count": 152, "max_ms": 92.754, "p95_ms": 89.85}, "visualize": {"avg_ms": 8.52, "count": 152, "max_ms": 10.369, "p95_ms": 9.703}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.888, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2919}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4376.172, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 213, "h264_decode_error_count": 395, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:51:41+09:00", "last_error_summary": "[h264 @ 0x5576dbd98d00] error while decoding MB 29 29, bytestream -5", "last_exit_at": "2026-07-02T04:51:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.642, "last_started_at": "2026-07-02T04:49:31+09:00", "pid": 38350, "process": {"exists": false, "pid": 38350}, "restart_count": 238, "restart_count_per_hour": 8, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 239}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 201.873, "sample_count": 1796, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 350392, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:51:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T04:51:41+09:00", "last_frame_read_age_sec": 0.187, "last_frame_read_at": "2026-07-02T04:51:41+09:00", "mjpeg": {"age_sec": 0.18, "exists": true, "mtime": "2026-07-02T04:51:41.156338+09:00", "mtime_epoch": 1782935501.1563385, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 123778}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:51:41+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1040.816, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 152, "pushed_frames": 152, "read_frames": 151, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.085, "pushed": 5.085, "read": 5.052}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.811, "count": 152, "max_ms": 8.393, "p95_ms": 7.423}, "gstreamer_push": {"avg_ms": 0.933, "count": 152, "max_ms": 2.903, "p95_ms": 1.142}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.969, "count": 152, "max_ms": 71.756, "p95_ms": 67.492}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.606, "count": 152, "max_ms": 92.742, "p95_ms": 89.836}, "yolo11_640_fire_11class": {"avg_ms": 54.736, "count": 152, "max_ms": 72.927, "p95_ms": 70.549}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.075, "count": 152, "max_ms": 90.688, "p95_ms": 87.333}}, "truncated_count": 0}, "processing_total": {"avg_ms": 187.671, "count": 152, "max_ms": 249.196, "p95_ms": 216.181}, "read_interval": {"avg_ms": 197.901, "count": 151, "max_ms": 260.665, "p95_ms": 227.485}, "triton_wait": {"avg_ms": 68.908, "count": 152, "max_ms": 92.754, "p95_ms": 89.85}, "visualize": {"avg_ms": 8.52, "count": 152, "max_ms": 10.369, "p95_ms": 9.703}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.891, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2919}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4486.676, "limit_mb": 8796093022207.996}, "detail": {"pid": 38457, "restart_count": 239, "start_count": 240}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 213, "h264_decode_error_count": 395, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:51:41+09:00", "last_error_summary": "[h264 @ 0x5576dbd98d00] error while decoding MB 29 29, bytestream -5", "last_exit_at": "2026-07-02T04:51:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T04:51:44+09:00", "pid": 38457, "process": {"exists": true, "fd_count": 5, "hwm_mb": 147.293, "name": "ffmpeg", "pid": 38457, "rss_mb": 147.293, "state": "S (sleeping)", "threads": 106}, "restart_count": 239, "restart_count_per_hour": 9, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 240}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 202.139, "sample_count": 1774, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 350392, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:51:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.701, "last_frame_pushed_at": "2026-07-02T04:51:41+09:00", "last_frame_read_age_sec": 4.881, "last_frame_read_at": "2026-07-02T04:51:41+09:00", "mjpeg": {"age_sec": 4.686, "exists": true, "mtime": "2026-07-02T04:51:41.344342+09:00", "mtime_epoch": 1782935501.3443415, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 124106}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:51:46+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1040.816, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 128, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.309, "pushed": 4.309, "read": 4.275}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.76, "count": 128, "max_ms": 8.083, "p95_ms": 7.352}, "gstreamer_push": {"avg_ms": 0.937, "count": 129, "max_ms": 2.903, "p95_ms": 1.142}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.935, "count": 128, "max_ms": 69.611, "p95_ms": 66.661}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.756, "count": 128, "max_ms": 92.742, "p95_ms": 89.802}, "yolo11_640_fire_11class": {"avg_ms": 54.906, "count": 128, "max_ms": 72.927, "p95_ms": 70.193}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.212, "count": 128, "max_ms": 90.688, "p95_ms": 87.07}}, "truncated_count": 0}, "processing_total": {"avg_ms": 187.151, "count": 129, "max_ms": 249.196, "p95_ms": 216.635}, "read_interval": {"avg_ms": 197.406, "count": 128, "max_ms": 260.665, "p95_ms": 227.485}, "triton_wait": {"avg_ms": 69.062, "count": 128, "max_ms": 92.754, "p95_ms": 89.815}, "visualize": {"avg_ms": 8.477, "count": 128, "max_ms": 10.315, "p95_ms": 9.535}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.94, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2925}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4375.914, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55b1844179c0] error while decoding MB 35 17, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 214, "h264_decode_error_count": 396, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:53:48+09:00", "last_error_summary": "[h264 @ 0x55b1844179c0] error while decoding MB 35 17, bytestream -5", "last_exit_at": "2026-07-02T04:51:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.729, "last_started_at": "2026-07-02T04:51:44+09:00", "pid": 38457, "process": {"exists": false, "pid": 38457}, "restart_count": 239, "restart_count_per_hour": 9, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 240}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 211.284, "sample_count": 1581, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 351028, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:53:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-02T04:53:48+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-02T04:53:48+09:00", "mjpeg": {"age_sec": 0.048, "exists": true, "mtime": "2026-07-02T04:53:48.710409+09:00", "mtime_epoch": 1782935628.7104094, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 123563}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:53:48+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1038.18, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 151, "pushed_frames": 151, "read_frames": 151, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.066, "pushed": 5.066, "read": 5.066}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.714, "count": 151, "max_ms": 10.198, "p95_ms": 7.467}, "gstreamer_push": {"avg_ms": 0.972, "count": 151, "max_ms": 2.926, "p95_ms": 1.8}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.862, "count": 151, "max_ms": 70.614, "p95_ms": 66.256}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.884, "count": 151, "max_ms": 95.084, "p95_ms": 89.333}, "yolo11_640_fire_11class": {"avg_ms": 54.0, "count": 151, "max_ms": 75.221, "p95_ms": 69.163}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.854, "count": 151, "max_ms": 92.75, "p95_ms": 88.243}}, "truncated_count": 0}, "processing_total": {"avg_ms": 188.254, "count": 151, "max_ms": 251.851, "p95_ms": 217.706}, "read_interval": {"avg_ms": 198.339, "count": 151, "max_ms": 261.617, "p95_ms": 227.278}, "triton_wait": {"avg_ms": 69.152, "count": 151, "max_ms": 95.096, "p95_ms": 89.343}, "visualize": {"avg_ms": 8.58, "count": 151, "max_ms": 12.19, "p95_ms": 10.122}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.805, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2925}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4374.938, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T04:53:48+09:00", "last_error_summary": "[h264 @ 0x55b1844179c0] error while decoding MB 35 17, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 214, "h264_decode_error_count": 396, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:53:48+09:00", "last_error_summary": "[h264 @ 0x55b1844179c0] error while decoding MB 35 17, bytestream -5", "last_exit_at": "2026-07-02T04:53:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.885, "last_started_at": "2026-07-02T04:51:44+09:00", "pid": 38457, "process": {"exists": false, "pid": 38457}, "restart_count": 239, "restart_count_per_hour": 9, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 240}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 211.284, "sample_count": 1581, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 351029, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:53:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T04:53:48+09:00", "last_frame_read_age_sec": 0.21, "last_frame_read_at": "2026-07-02T04:53:48+09:00", "mjpeg": {"age_sec": 0.204, "exists": true, "mtime": "2026-07-02T04:53:48.710409+09:00", "mtime_epoch": 1782935628.7104094, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 123563}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:53:48+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1038.18, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 152, "pushed_frames": 152, "read_frames": 151, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.073, "pushed": 5.073, "read": 5.04}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.71, "count": 151, "max_ms": 10.198, "p95_ms": 7.467}, "gstreamer_push": {"avg_ms": 0.971, "count": 152, "max_ms": 2.926, "p95_ms": 1.8}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.799, "count": 151, "max_ms": 70.614, "p95_ms": 66.256}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.82, "count": 151, "max_ms": 95.084, "p95_ms": 89.333}, "yolo11_640_fire_11class": {"avg_ms": 53.952, "count": 151, "max_ms": 75.221, "p95_ms": 69.163}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.806, "count": 151, "max_ms": 92.75, "p95_ms": 88.243}}, "truncated_count": 0}, "processing_total": {"avg_ms": 188.355, "count": 152, "max_ms": 251.851, "p95_ms": 217.706}, "read_interval": {"avg_ms": 198.339, "count": 151, "max_ms": 261.617, "p95_ms": 227.278}, "triton_wait": {"avg_ms": 69.087, "count": 151, "max_ms": 95.096, "p95_ms": 89.343}, "visualize": {"avg_ms": 8.591, "count": 151, "max_ms": 12.19, "p95_ms": 10.731}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.961, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2925}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4374.953, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 214, "h264_decode_error_count": 396, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:53:48+09:00", "last_error_summary": "[h264 @ 0x55b1844179c0] error while decoding MB 35 17, bytestream -5", "last_exit_at": "2026-07-02T04:53:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.888, "last_started_at": "2026-07-02T04:51:44+09:00", "pid": 38457, "process": {"exists": false, "pid": 38457}, "restart_count": 239, "restart_count_per_hour": 9, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 240}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 211.284, "sample_count": 1581, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 351029, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:53:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-02T04:53:48+09:00", "last_frame_read_age_sec": 0.213, "last_frame_read_at": "2026-07-02T04:53:48+09:00", "mjpeg": {"age_sec": 0.207, "exists": true, "mtime": "2026-07-02T04:53:48.710409+09:00", "mtime_epoch": 1782935628.7104094, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 123563}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:53:48+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1038.18, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 152, "pushed_frames": 152, "read_frames": 151, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.073, "pushed": 5.073, "read": 5.039}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.71, "count": 151, "max_ms": 10.198, "p95_ms": 7.467}, "gstreamer_push": {"avg_ms": 0.971, "count": 152, "max_ms": 2.926, "p95_ms": 1.8}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.799, "count": 151, "max_ms": 70.614, "p95_ms": 66.256}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.82, "count": 151, "max_ms": 95.084, "p95_ms": 89.333}, "yolo11_640_fire_11class": {"avg_ms": 53.952, "count": 151, "max_ms": 75.221, "p95_ms": 69.163}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.806, "count": 151, "max_ms": 92.75, "p95_ms": 88.243}}, "truncated_count": 0}, "processing_total": {"avg_ms": 188.355, "count": 152, "max_ms": 251.851, "p95_ms": 217.706}, "read_interval": {"avg_ms": 198.339, "count": 151, "max_ms": 261.617, "p95_ms": 227.278}, "triton_wait": {"avg_ms": 69.087, "count": 151, "max_ms": 95.096, "p95_ms": 89.343}, "visualize": {"avg_ms": 8.591, "count": 151, "max_ms": 12.19, "p95_ms": 10.731}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.964, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2925}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4490.906, "limit_mb": 8796093022207.996}, "detail": {"pid": 38564, "restart_count": 240, "start_count": 241}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 214, "h264_decode_error_count": 396, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:53:48+09:00", "last_error_summary": "[h264 @ 0x55b1844179c0] error while decoding MB 35 17, bytestream -5", "last_exit_at": "2026-07-02T04:53:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T04:53:51+09:00", "pid": 38564, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.707, "name": "ffmpeg", "pid": 38564, "rss_mb": 141.707, "state": "S (sleeping)", "threads": 106}, "restart_count": 240, "restart_count_per_hour": 10, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 241}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 211.284, "sample_count": 1552, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 351029, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:53:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.968, "last_frame_pushed_at": "2026-07-02T04:53:48+09:00", "last_frame_read_age_sec": 5.173, "last_frame_read_at": "2026-07-02T04:53:48+09:00", "mjpeg": {"age_sec": 4.951, "exists": true, "mtime": "2026-07-02T04:53:48.926413+09:00", "mtime_epoch": 1782935628.9264128, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 123409}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:53:53+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1038.18, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 127, "pushed_frames": 127, "read_frames": 126, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.245, "pushed": 4.245, "read": 4.212}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.726, "count": 126, "max_ms": 8.138, "p95_ms": 7.457}, "gstreamer_push": {"avg_ms": 0.975, "count": 127, "max_ms": 2.926, "p95_ms": 1.8}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.781, "count": 126, "max_ms": 69.9, "p95_ms": 65.919}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.27, "count": 126, "max_ms": 95.084, "p95_ms": 88.198}, "yolo11_640_fire_11class": {"avg_ms": 53.569, "count": 126, "max_ms": 75.221, "p95_ms": 68.847}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.417, "count": 126, "max_ms": 92.75, "p95_ms": 87.742}}, "truncated_count": 0}, "processing_total": {"avg_ms": 187.771, "count": 127, "max_ms": 250.298, "p95_ms": 214.523}, "read_interval": {"avg_ms": 197.69, "count": 126, "max_ms": 260.674, "p95_ms": 224.045}, "triton_wait": {"avg_ms": 68.497, "count": 126, "max_ms": 95.096, "p95_ms": 88.269}, "visualize": {"avg_ms": 8.524, "count": 126, "max_ms": 11.782, "p95_ms": 9.215}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.916, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2930}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4371.969, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55b4c0830100] error while decoding MB 4 24, bytestream -38", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 215, "h264_decode_error_count": 397, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:55:53+09:00", "last_error_summary": "[h264 @ 0x55b4c0830100] error while decoding MB 4 24, bytestream -38", "last_exit_at": "2026-07-02T04:53:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.961, "last_started_at": "2026-07-02T04:53:51+09:00", "pid": 38564, "process": {"exists": false, "pid": 38564}, "restart_count": 240, "restart_count_per_hour": 10, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 241}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 214.147, "sample_count": 1482, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 351627, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:55:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-02T04:55:53+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-02T04:55:53+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-02T04:55:53.796457+09:00", "mtime_epoch": 1782935753.796457, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 148115}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:55:53+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1036.715, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 149, "pushed_frames": 149, "read_frames": 149, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.999, "pushed": 4.999, "read": 4.999}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.81, "count": 149, "max_ms": 7.844, "p95_ms": 7.27}, "gstreamer_push": {"avg_ms": 0.905, "count": 149, "max_ms": 1.937, "p95_ms": 1.094}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.539, "count": 149, "max_ms": 71.331, "p95_ms": 62.403}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.04, "count": 149, "max_ms": 90.753, "p95_ms": 86.941}, "yolo11_640_fire_11class": {"avg_ms": 53.313, "count": 149, "max_ms": 72.3, "p95_ms": 67.914}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.809, "count": 149, "max_ms": 89.663, "p95_ms": 86.166}}, "truncated_count": 0}, "processing_total": {"avg_ms": 191.63, "count": 149, "max_ms": 254.168, "p95_ms": 214.662}, "read_interval": {"avg_ms": 201.391, "count": 149, "max_ms": 264.814, "p95_ms": 223.752}, "triton_wait": {"avg_ms": 67.439, "count": 149, "max_ms": 90.764, "p95_ms": 87.654}, "visualize": {"avg_ms": 8.806, "count": 149, "max_ms": 13.455, "p95_ms": 9.553}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.804, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2930}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4371.215, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T04:55:53+09:00", "last_error_summary": "[h264 @ 0x55b4c0830100] error while decoding MB 4 24, bytestream -38", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 215, "h264_decode_error_count": 397, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:55:53+09:00", "last_error_summary": "[h264 @ 0x55b4c0830100] error while decoding MB 4 24, bytestream -38", "last_exit_at": "2026-07-02T04:55:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 122.101, "last_started_at": "2026-07-02T04:53:51+09:00", "pid": 38564, "process": {"exists": false, "pid": 38564}, "restart_count": 240, "restart_count_per_hour": 10, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 241}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 214.147, "sample_count": 1482, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 351628, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:55:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T04:55:53+09:00", "last_frame_read_age_sec": 0.19, "last_frame_read_at": "2026-07-02T04:55:53+09:00", "mjpeg": {"age_sec": 0.181, "exists": true, "mtime": "2026-07-02T04:55:53.796457+09:00", "mtime_epoch": 1782935753.796457, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 148115}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:55:53+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1036.715, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 150, "pushed_frames": 150, "read_frames": 149, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.01, "pushed": 5.01, "read": 4.976}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.816, "count": 149, "max_ms": 7.969, "p95_ms": 7.426}, "gstreamer_push": {"avg_ms": 0.904, "count": 150, "max_ms": 1.937, "p95_ms": 1.094}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.365, "count": 149, "max_ms": 71.331, "p95_ms": 61.622}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.856, "count": 149, "max_ms": 90.753, "p95_ms": 86.941}, "yolo11_640_fire_11class": {"avg_ms": 53.176, "count": 149, "max_ms": 72.3, "p95_ms": 67.914}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.623, "count": 149, "max_ms": 89.663, "p95_ms": 86.166}}, "truncated_count": 0}, "processing_total": {"avg_ms": 191.59, "count": 150, "max_ms": 254.168, "p95_ms": 214.662}, "read_interval": {"avg_ms": 201.391, "count": 149, "max_ms": 264.814, "p95_ms": 223.752}, "triton_wait": {"avg_ms": 67.255, "count": 149, "max_ms": 90.764, "p95_ms": 87.654}, "visualize": {"avg_ms": 8.806, "count": 149, "max_ms": 13.455, "p95_ms": 9.553}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.943, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2930}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4371.242, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 215, "h264_decode_error_count": 397, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:55:53+09:00", "last_error_summary": "[h264 @ 0x55b4c0830100] error while decoding MB 4 24, bytestream -38", "last_exit_at": "2026-07-02T04:55:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 122.105, "last_started_at": "2026-07-02T04:53:51+09:00", "pid": 38564, "process": {"exists": false, "pid": 38564}, "restart_count": 240, "restart_count_per_hour": 10, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 241}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 214.147, "sample_count": 1482, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 351628, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:55:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T04:55:53+09:00", "last_frame_read_age_sec": 0.193, "last_frame_read_at": "2026-07-02T04:55:53+09:00", "mjpeg": {"age_sec": 0.185, "exists": true, "mtime": "2026-07-02T04:55:53.796457+09:00", "mtime_epoch": 1782935753.796457, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 148115}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:55:53+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1036.715, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 150, "pushed_frames": 150, "read_frames": 149, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.009, "pushed": 5.009, "read": 4.976}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.816, "count": 149, "max_ms": 7.969, "p95_ms": 7.426}, "gstreamer_push": {"avg_ms": 0.904, "count": 150, "max_ms": 1.937, "p95_ms": 1.094}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.365, "count": 149, "max_ms": 71.331, "p95_ms": 61.622}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.856, "count": 149, "max_ms": 90.753, "p95_ms": 86.941}, "yolo11_640_fire_11class": {"avg_ms": 53.176, "count": 149, "max_ms": 72.3, "p95_ms": 67.914}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.623, "count": 149, "max_ms": 89.663, "p95_ms": 86.166}}, "truncated_count": 0}, "processing_total": {"avg_ms": 191.59, "count": 150, "max_ms": 254.168, "p95_ms": 214.662}, "read_interval": {"avg_ms": 201.391, "count": 149, "max_ms": 264.814, "p95_ms": 223.752}, "triton_wait": {"avg_ms": 67.255, "count": 149, "max_ms": 90.764, "p95_ms": 87.654}, "visualize": {"avg_ms": 8.806, "count": 149, "max_ms": 13.455, "p95_ms": 9.553}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.946, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2930}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4482.062, "limit_mb": 8796093022207.996}, "detail": {"pid": 38671, "restart_count": 241, "start_count": 242}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 215, "h264_decode_error_count": 397, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:55:53+09:00", "last_error_summary": "[h264 @ 0x55b4c0830100] error while decoding MB 4 24, bytestream -38", "last_exit_at": "2026-07-02T04:55:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T04:55:56+09:00", "pid": 38671, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.34, "name": "ffmpeg", "pid": 38671, "rss_mb": 139.34, "state": "S (sleeping)", "threads": 106}, "restart_count": 241, "restart_count_per_hour": 11, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 242}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 214.155, "sample_count": 1457, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 351628, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:55:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.399, "last_frame_pushed_at": "2026-07-02T04:55:53+09:00", "last_frame_read_age_sec": 4.586, "last_frame_read_at": "2026-07-02T04:55:53+09:00", "mjpeg": {"age_sec": 4.386, "exists": true, "mtime": "2026-07-02T04:55:53.988460+09:00", "mtime_epoch": 1782935753.9884603, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 148346}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:55:58+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1036.715, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 127, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.272, "pushed": 4.272, "read": 4.239}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.845, "count": 127, "max_ms": 7.969, "p95_ms": 7.426}, "gstreamer_push": {"avg_ms": 0.896, "count": 128, "max_ms": 1.937, "p95_ms": 1.016}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.402, "count": 127, "max_ms": 66.659, "p95_ms": 61.045}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.113, "count": 127, "max_ms": 90.753, "p95_ms": 86.941}, "yolo11_640_fire_11class": {"avg_ms": 53.486, "count": 127, "max_ms": 70.272, "p95_ms": 67.914}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.884, "count": 127, "max_ms": 89.663, "p95_ms": 86.166}}, "truncated_count": 0}, "processing_total": {"avg_ms": 191.505, "count": 128, "max_ms": 248.024, "p95_ms": 212.934}, "read_interval": {"avg_ms": 201.274, "count": 127, "max_ms": 257.301, "p95_ms": 222.246}, "triton_wait": {"avg_ms": 67.515, "count": 127, "max_ms": 90.764, "p95_ms": 86.953}, "visualize": {"avg_ms": 8.816, "count": 127, "max_ms": 13.455, "p95_ms": 9.553}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.963, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2935}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4380.52, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55c7b0e33340] error while decoding MB 26 30, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 216, "h264_decode_error_count": 398, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:58:00+09:00", "last_error_summary": "[h264 @ 0x55c7b0e33340] error while decoding MB 26 30, bytestream -5", "last_exit_at": "2026-07-02T04:55:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.142, "last_started_at": "2026-07-02T04:55:56+09:00", "pid": 38671, "process": {"exists": false, "pid": 38671}, "restart_count": 241, "restart_count_per_hour": 10, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 242}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 214.523, "sample_count": 1454, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 352232, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:58:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-02T04:58:00+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-02T04:58:00+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-02T04:58:00.470544+09:00", "mtime_epoch": 1782935880.4705439, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 137159}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:58:00+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1039.504, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 149, "pushed_frames": 149, "read_frames": 149, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.973, "pushed": 4.973, "read": 4.973}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.855, "count": 148, "max_ms": 10.322, "p95_ms": 7.378}, "gstreamer_push": {"avg_ms": 0.903, "count": 149, "max_ms": 2.034, "p95_ms": 0.924}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.822, "count": 148, "max_ms": 71.757, "p95_ms": 60.543}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.238, "count": 148, "max_ms": 91.418, "p95_ms": 88.418}, "yolo11_640_fire_11class": {"avg_ms": 53.714, "count": 148, "max_ms": 71.401, "p95_ms": 69.571}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.024, "count": 148, "max_ms": 91.296, "p95_ms": 87.462}}, "truncated_count": 0}, "processing_total": {"avg_ms": 192.194, "count": 149, "max_ms": 256.675, "p95_ms": 215.722}, "read_interval": {"avg_ms": 201.988, "count": 149, "max_ms": 268.514, "p95_ms": 225.038}, "triton_wait": {"avg_ms": 67.636, "count": 148, "max_ms": 91.43, "p95_ms": 89.191}, "visualize": {"avg_ms": 8.864, "count": 148, "max_ms": 11.543, "p95_ms": 9.772}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.965, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2935}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4379.594, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T04:58:00+09:00", "last_error_summary": "[h264 @ 0x55c7b0e33340] error while decoding MB 26 30, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 216, "h264_decode_error_count": 398, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:58:00+09:00", "last_error_summary": "[h264 @ 0x55c7b0e33340] error while decoding MB 26 30, bytestream -5", "last_exit_at": "2026-07-02T04:58:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.301, "last_started_at": "2026-07-02T04:55:56+09:00", "pid": 38671, "process": {"exists": false, "pid": 38671}, "restart_count": 241, "restart_count_per_hour": 10, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 242}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 214.523, "sample_count": 1454, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 352233, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:58:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T04:58:00+09:00", "last_frame_read_age_sec": 0.211, "last_frame_read_at": "2026-07-02T04:58:00+09:00", "mjpeg": {"age_sec": 0.204, "exists": true, "mtime": "2026-07-02T04:58:00.470544+09:00", "mtime_epoch": 1782935880.4705439, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 137159}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:58:00+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1039.504, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 149, "pushed_frames": 149, "read_frames": 148, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.983, "pushed": 4.983, "read": 4.949}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.857, "count": 148, "max_ms": 10.322, "p95_ms": 7.378}, "gstreamer_push": {"avg_ms": 0.904, "count": 149, "max_ms": 2.034, "p95_ms": 0.924}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.777, "count": 148, "max_ms": 71.757, "p95_ms": 60.405}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.146, "count": 148, "max_ms": 91.418, "p95_ms": 88.418}, "yolo11_640_fire_11class": {"avg_ms": 53.723, "count": 148, "max_ms": 71.401, "p95_ms": 69.571}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.969, "count": 148, "max_ms": 91.296, "p95_ms": 87.196}}, "truncated_count": 0}, "processing_total": {"avg_ms": 192.345, "count": 149, "max_ms": 256.675, "p95_ms": 215.722}, "read_interval": {"avg_ms": 202.048, "count": 148, "max_ms": 268.514, "p95_ms": 225.038}, "triton_wait": {"avg_ms": 67.582, "count": 148, "max_ms": 91.43, "p95_ms": 89.191}, "visualize": {"avg_ms": 8.866, "count": 148, "max_ms": 11.543, "p95_ms": 9.772}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.903, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2935}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4379.586, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 216, "h264_decode_error_count": 398, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:58:00+09:00", "last_error_summary": "[h264 @ 0x55c7b0e33340] error while decoding MB 26 30, bytestream -5", "last_exit_at": "2026-07-02T04:58:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.305, "last_started_at": "2026-07-02T04:55:56+09:00", "pid": 38671, "process": {"exists": false, "pid": 38671}, "restart_count": 241, "restart_count_per_hour": 10, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 242}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 214.523, "sample_count": 1454, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 352233, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:58:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T04:58:00+09:00", "last_frame_read_age_sec": 0.214, "last_frame_read_at": "2026-07-02T04:58:00+09:00", "mjpeg": {"age_sec": 0.207, "exists": true, "mtime": "2026-07-02T04:58:00.470544+09:00", "mtime_epoch": 1782935880.4705439, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 137159}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:58:00+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1039.504, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 149, "pushed_frames": 149, "read_frames": 148, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.982, "pushed": 4.982, "read": 4.949}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.857, "count": 148, "max_ms": 10.322, "p95_ms": 7.378}, "gstreamer_push": {"avg_ms": 0.904, "count": 149, "max_ms": 2.034, "p95_ms": 0.924}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.777, "count": 148, "max_ms": 71.757, "p95_ms": 60.405}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.146, "count": 148, "max_ms": 91.418, "p95_ms": 88.418}, "yolo11_640_fire_11class": {"avg_ms": 53.723, "count": 148, "max_ms": 71.401, "p95_ms": 69.571}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.969, "count": 148, "max_ms": 91.296, "p95_ms": 87.196}}, "truncated_count": 0}, "processing_total": {"avg_ms": 192.345, "count": 149, "max_ms": 256.675, "p95_ms": 215.722}, "read_interval": {"avg_ms": 202.048, "count": 148, "max_ms": 268.514, "p95_ms": 225.038}, "triton_wait": {"avg_ms": 67.582, "count": 148, "max_ms": 91.43, "p95_ms": 89.191}, "visualize": {"avg_ms": 8.866, "count": 148, "max_ms": 11.543, "p95_ms": 9.772}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.907, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2935}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4487.41, "limit_mb": 8796093022207.996}, "detail": {"pid": 38778, "restart_count": 242, "start_count": 243}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 216, "h264_decode_error_count": 398, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T04:58:00+09:00", "last_error_summary": "[h264 @ 0x55c7b0e33340] error while decoding MB 26 30, bytestream -5", "last_exit_at": "2026-07-02T04:58:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T04:58:03+09:00", "pid": 38778, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.082, "name": "ffmpeg", "pid": 38778, "rss_mb": 139.082, "state": "S (sleeping)", "threads": 106}, "restart_count": 242, "restart_count_per_hour": 11, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 243}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 214.62, "sample_count": 1429, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 352233, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T04:58:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.552, "last_frame_pushed_at": "2026-07-02T04:58:00+09:00", "last_frame_read_age_sec": 4.76, "last_frame_read_at": "2026-07-02T04:58:00+09:00", "mjpeg": {"age_sec": 4.537, "exists": true, "mtime": "2026-07-02T04:58:00.686548+09:00", "mtime_epoch": 1782935880.6865475, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 139744}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T04:58:05+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1039.504, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 126, "pushed_frames": 126, "read_frames": 125, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.217, "pushed": 4.217, "read": 4.183}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.843, "count": 126, "max_ms": 8.318, "p95_ms": 7.292}, "gstreamer_push": {"avg_ms": 0.904, "count": 126, "max_ms": 2.034, "p95_ms": 0.924}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.746, "count": 126, "max_ms": 71.757, "p95_ms": 59.118}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.549, "count": 126, "max_ms": 91.418, "p95_ms": 88.443}, "yolo11_640_fire_11class": {"avg_ms": 53.986, "count": 126, "max_ms": 71.401, "p95_ms": 69.824}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.298, "count": 126, "max_ms": 91.296, "p95_ms": 87.462}}, "truncated_count": 0}, "processing_total": {"avg_ms": 192.79, "count": 126, "max_ms": 256.675, "p95_ms": 217.196}, "read_interval": {"avg_ms": 202.484, "count": 125, "max_ms": 268.514, "p95_ms": 226.643}, "triton_wait": {"avg_ms": 67.997, "count": 126, "max_ms": 91.43, "p95_ms": 89.198}, "visualize": {"avg_ms": 8.84, "count": 126, "max_ms": 11.543, "p95_ms": 9.498}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.881, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2940}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4381.508, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55f3c2b4b6c0] error while decoding MB 61 41, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 217, "h264_decode_error_count": 399, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:00:05+09:00", "last_error_summary": "[h264 @ 0x55f3c2b4b6c0] error while decoding MB 61 41, bytestream -5", "last_exit_at": "2026-07-02T04:58:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 122.583, "last_started_at": "2026-07-02T04:58:03+09:00", "pid": 38778, "process": {"exists": false, "pid": 38778}, "restart_count": 242, "restart_count_per_hour": 11, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 243}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 216.473, "sample_count": 1437, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 352826, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:00:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-02T05:00:05+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-02T05:00:05+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-02T05:00:05.760618+09:00", "mtime_epoch": 1782936005.7606182, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 136754}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:00:05+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1038.68, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 148, "pushed_frames": 148, "read_frames": 148, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.952, "pushed": 4.952, "read": 4.952}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.03, "count": 148, "max_ms": 9.271, "p95_ms": 7.941}, "gstreamer_push": {"avg_ms": 0.918, "count": 148, "max_ms": 2.924, "p95_ms": 1.426}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.09, "count": 148, "max_ms": 71.175, "p95_ms": 62.77}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.159, "count": 148, "max_ms": 96.707, "p95_ms": 87.07}, "yolo11_640_fire_11class": {"avg_ms": 55.451, "count": 148, "max_ms": 79.448, "p95_ms": 72.726}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.057, "count": 148, "max_ms": 95.925, "p95_ms": 88.578}}, "truncated_count": 0}, "processing_total": {"avg_ms": 193.295, "count": 148, "max_ms": 247.061, "p95_ms": 216.839}, "read_interval": {"avg_ms": 202.99, "count": 148, "max_ms": 257.481, "p95_ms": 226.15}, "triton_wait": {"avg_ms": 68.794, "count": 148, "max_ms": 96.72, "p95_ms": 88.59}, "visualize": {"avg_ms": 8.877, "count": 148, "max_ms": 11.45, "p95_ms": 9.992}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.885, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2940}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4381.07, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T05:00:05+09:00", "last_error_summary": "[h264 @ 0x55f3c2b4b6c0] error while decoding MB 61 41, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 217, "h264_decode_error_count": 399, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:00:05+09:00", "last_error_summary": "[h264 @ 0x55f3c2b4b6c0] error while decoding MB 61 41, bytestream -5", "last_exit_at": "2026-07-02T05:00:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 122.724, "last_started_at": "2026-07-02T04:58:03+09:00", "pid": 38778, "process": {"exists": false, "pid": 38778}, "restart_count": 242, "restart_count_per_hour": 11, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 243}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 216.473, "sample_count": 1437, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 352827, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:00:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T05:00:05+09:00", "last_frame_read_age_sec": 0.194, "last_frame_read_at": "2026-07-02T05:00:05+09:00", "mjpeg": {"age_sec": 0.187, "exists": true, "mtime": "2026-07-02T05:00:05.760618+09:00", "mtime_epoch": 1782936005.7606182, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 136754}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:00:05+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1038.68, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 148, "pushed_frames": 148, "read_frames": 147, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.962, "pushed": 4.962, "read": 4.928}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.025, "count": 148, "max_ms": 9.271, "p95_ms": 7.941}, "gstreamer_push": {"avg_ms": 0.918, "count": 148, "max_ms": 2.924, "p95_ms": 1.426}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.075, "count": 148, "max_ms": 71.175, "p95_ms": 62.77}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.968, "count": 148, "max_ms": 96.707, "p95_ms": 87.07}, "yolo11_640_fire_11class": {"avg_ms": 55.278, "count": 148, "max_ms": 79.448, "p95_ms": 72.726}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.829, "count": 148, "max_ms": 95.925, "p95_ms": 88.578}}, "truncated_count": 0}, "processing_total": {"avg_ms": 193.15, "count": 148, "max_ms": 247.061, "p95_ms": 216.839}, "read_interval": {"avg_ms": 202.859, "count": 147, "max_ms": 257.481, "p95_ms": 226.15}, "triton_wait": {"avg_ms": 68.598, "count": 148, "max_ms": 96.72, "p95_ms": 88.59}, "visualize": {"avg_ms": 8.889, "count": 148, "max_ms": 11.45, "p95_ms": 10.13}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.828, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2940}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4381.0, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 217, "h264_decode_error_count": 399, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:00:05+09:00", "last_error_summary": "[h264 @ 0x55f3c2b4b6c0] error while decoding MB 61 41, bytestream -5", "last_exit_at": "2026-07-02T05:00:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 122.728, "last_started_at": "2026-07-02T04:58:03+09:00", "pid": 38778, "process": {"exists": false, "pid": 38778}, "restart_count": 242, "restart_count_per_hour": 11, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 243}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 216.473, "sample_count": 1437, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 352827, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:00:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T05:00:05+09:00", "last_frame_read_age_sec": 0.197, "last_frame_read_at": "2026-07-02T05:00:05+09:00", "mjpeg": {"age_sec": 0.19, "exists": true, "mtime": "2026-07-02T05:00:05.760618+09:00", "mtime_epoch": 1782936005.7606182, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 136754}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:00:05+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1038.68, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 148, "pushed_frames": 148, "read_frames": 147, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.961, "pushed": 4.961, "read": 4.928}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.025, "count": 148, "max_ms": 9.271, "p95_ms": 7.941}, "gstreamer_push": {"avg_ms": 0.918, "count": 148, "max_ms": 2.924, "p95_ms": 1.426}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.075, "count": 148, "max_ms": 71.175, "p95_ms": 62.77}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.968, "count": 148, "max_ms": 96.707, "p95_ms": 87.07}, "yolo11_640_fire_11class": {"avg_ms": 55.278, "count": 148, "max_ms": 79.448, "p95_ms": 72.726}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.829, "count": 148, "max_ms": 95.925, "p95_ms": 88.578}}, "truncated_count": 0}, "processing_total": {"avg_ms": 193.15, "count": 148, "max_ms": 247.061, "p95_ms": 216.839}, "read_interval": {"avg_ms": 202.859, "count": 147, "max_ms": 257.481, "p95_ms": 226.15}, "triton_wait": {"avg_ms": 68.598, "count": 148, "max_ms": 96.72, "p95_ms": 88.59}, "visualize": {"avg_ms": 8.889, "count": 148, "max_ms": 11.45, "p95_ms": 10.13}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.831, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2940}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4489.453, "limit_mb": 8796093022207.996}, "detail": {"pid": 38885, "restart_count": 243, "start_count": 244}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 217, "h264_decode_error_count": 399, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:00:05+09:00", "last_error_summary": "[h264 @ 0x55f3c2b4b6c0] error while decoding MB 61 41, bytestream -5", "last_exit_at": "2026-07-02T05:00:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T05:00:07+09:00", "pid": 38885, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.844, "name": "ffmpeg", "pid": 38885, "rss_mb": 138.844, "state": "S (sleeping)", "threads": 106}, "restart_count": 243, "restart_count_per_hour": 12, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 244}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 216.661, "sample_count": 1417, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 352827, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:00:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.03, "last_frame_pushed_at": "2026-07-02T05:00:05+09:00", "last_frame_read_age_sec": 4.221, "last_frame_read_at": "2026-07-02T05:00:05+09:00", "mjpeg": {"age_sec": 4.017, "exists": true, "mtime": "2026-07-02T05:00:05.956621+09:00", "mtime_epoch": 1782936005.9566214, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 136563}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:00:09+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1038.68, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 127, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.289, "pushed": 4.289, "read": 4.256}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.088, "count": 128, "max_ms": 9.271, "p95_ms": 7.995}, "gstreamer_push": {"avg_ms": 0.919, "count": 128, "max_ms": 2.924, "p95_ms": 1.426}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.246, "count": 128, "max_ms": 71.175, "p95_ms": 62.77}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.93, "count": 128, "max_ms": 96.707, "p95_ms": 87.07}, "yolo11_640_fire_11class": {"avg_ms": 55.37, "count": 128, "max_ms": 79.448, "p95_ms": 72.726}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.807, "count": 128, "max_ms": 95.925, "p95_ms": 88.029}}, "truncated_count": 0}, "processing_total": {"avg_ms": 193.474, "count": 128, "max_ms": 247.061, "p95_ms": 218.21}, "read_interval": {"avg_ms": 203.187, "count": 127, "max_ms": 257.481, "p95_ms": 228.232}, "triton_wait": {"avg_ms": 68.597, "count": 128, "max_ms": 96.72, "p95_ms": 88.04}, "visualize": {"avg_ms": 8.908, "count": 128, "max_ms": 11.45, "p95_ms": 10.494}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.841, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2944}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4373.891, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56066918e680] error while decoding MB 46 8, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 218, "h264_decode_error_count": 400, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:01:59+09:00", "last_error_summary": "[h264 @ 0x56066918e680] error while decoding MB 46 8, bytestream -9", "last_exit_at": "2026-07-02T05:00:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.864, "last_started_at": "2026-07-02T05:00:07+09:00", "pid": 38885, "process": {"exists": false, "pid": 38885}, "restart_count": 243, "restart_count_per_hour": 12, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 244}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 237.652, "sample_count": 1361, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 353291, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:01:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-02T05:01:59+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-02T05:01:59+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-02T05:01:59.794513+09:00", "mtime_epoch": 1782936119.7945132, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165612}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:01:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1037.559, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 128, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.271, "pushed": 4.271, "read": 4.271}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.853, "count": 127, "max_ms": 16.154, "p95_ms": 10.801}, "gstreamer_push": {"avg_ms": 0.937, "count": 128, "max_ms": 2.824, "p95_ms": 1.837}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.51, "count": 127, "max_ms": 71.096, "p95_ms": 60.873}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.372, "count": 127, "max_ms": 91.71, "p95_ms": 83.579}, "yolo11_640_fire_11class": {"avg_ms": 53.053, "count": 127, "max_ms": 74.119, "p95_ms": 67.42}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.417, "count": 127, "max_ms": 91.501, "p95_ms": 84.944}}, "truncated_count": 0}, "processing_total": {"avg_ms": 225.746, "count": 128, "max_ms": 282.939, "p95_ms": 246.859}, "read_interval": {"avg_ms": 235.574, "count": 128, "max_ms": 293.52, "p95_ms": 257.501}, "triton_wait": {"avg_ms": 67.972, "count": 127, "max_ms": 91.722, "p95_ms": 87.358}, "visualize": {"avg_ms": 10.721, "count": 127, "max_ms": 14.99, "p95_ms": 12.092}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.973, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2944}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4375.641, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T05:01:59+09:00", "last_error_summary": "[h264 @ 0x56066918e680] error while decoding MB 46 8, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 218, "h264_decode_error_count": 400, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:01:59+09:00", "last_error_summary": "[h264 @ 0x56066918e680] error while decoding MB 46 8, bytestream -9", "last_exit_at": "2026-07-02T05:02:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.047, "last_started_at": "2026-07-02T05:00:07+09:00", "pid": 38885, "process": {"exists": false, "pid": 38885}, "restart_count": 243, "restart_count_per_hour": 12, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 244}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 237.652, "sample_count": 1361, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 353292, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:01:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T05:02:00+09:00", "last_frame_read_age_sec": 0.236, "last_frame_read_at": "2026-07-02T05:01:59+09:00", "mjpeg": {"age_sec": 0.226, "exists": true, "mtime": "2026-07-02T05:01:59.794513+09:00", "mtime_epoch": 1782936119.7945132, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165612}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:02:00+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1037.562, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 127, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.279, "pushed": 4.279, "read": 4.246}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.853, "count": 127, "max_ms": 16.154, "p95_ms": 10.801}, "gstreamer_push": {"avg_ms": 0.937, "count": 128, "max_ms": 2.824, "p95_ms": 1.837}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.46, "count": 127, "max_ms": 71.096, "p95_ms": 60.873}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.4, "count": 127, "max_ms": 91.71, "p95_ms": 83.579}, "yolo11_640_fire_11class": {"avg_ms": 53.083, "count": 127, "max_ms": 74.119, "p95_ms": 67.42}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.41, "count": 127, "max_ms": 91.501, "p95_ms": 84.944}}, "truncated_count": 0}, "processing_total": {"avg_ms": 225.734, "count": 128, "max_ms": 282.939, "p95_ms": 246.859}, "read_interval": {"avg_ms": 235.522, "count": 127, "max_ms": 293.52, "p95_ms": 257.501}, "triton_wait": {"avg_ms": 67.974, "count": 127, "max_ms": 91.722, "p95_ms": 87.358}, "visualize": {"avg_ms": 10.728, "count": 127, "max_ms": 14.99, "p95_ms": 12.092}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.914, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2944}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4375.633, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 218, "h264_decode_error_count": 400, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:01:59+09:00", "last_error_summary": "[h264 @ 0x56066918e680] error while decoding MB 46 8, bytestream -9", "last_exit_at": "2026-07-02T05:02:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 112.051, "last_started_at": "2026-07-02T05:00:07+09:00", "pid": 38885, "process": {"exists": false, "pid": 38885}, "restart_count": 243, "restart_count_per_hour": 12, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 244}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 237.652, "sample_count": 1361, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 353292, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:01:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T05:02:00+09:00", "last_frame_read_age_sec": 0.239, "last_frame_read_at": "2026-07-02T05:01:59+09:00", "mjpeg": {"age_sec": 0.229, "exists": true, "mtime": "2026-07-02T05:01:59.794513+09:00", "mtime_epoch": 1782936119.7945132, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165612}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:02:00+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1037.562, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 127, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.278, "pushed": 4.278, "read": 4.245}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.853, "count": 127, "max_ms": 16.154, "p95_ms": 10.801}, "gstreamer_push": {"avg_ms": 0.937, "count": 128, "max_ms": 2.824, "p95_ms": 1.837}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.46, "count": 127, "max_ms": 71.096, "p95_ms": 60.873}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.4, "count": 127, "max_ms": 91.71, "p95_ms": 83.579}, "yolo11_640_fire_11class": {"avg_ms": 53.083, "count": 127, "max_ms": 74.119, "p95_ms": 67.42}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.41, "count": 127, "max_ms": 91.501, "p95_ms": 84.944}}, "truncated_count": 0}, "processing_total": {"avg_ms": 225.734, "count": 128, "max_ms": 282.939, "p95_ms": 246.859}, "read_interval": {"avg_ms": 235.522, "count": 127, "max_ms": 293.52, "p95_ms": 257.501}, "triton_wait": {"avg_ms": 67.974, "count": 127, "max_ms": 91.722, "p95_ms": 87.358}, "visualize": {"avg_ms": 10.728, "count": 127, "max_ms": 14.99, "p95_ms": 12.092}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.917, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2944}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4487.152, "limit_mb": 8796093022207.996}, "detail": {"pid": 38992, "restart_count": 244, "start_count": 245}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 218, "h264_decode_error_count": 400, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:01:59+09:00", "last_error_summary": "[h264 @ 0x56066918e680] error while decoding MB 46 8, bytestream -9", "last_exit_at": "2026-07-02T05:02:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T05:02:02+09:00", "pid": 38992, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.148, "name": "ffmpeg", "pid": 38992, "rss_mb": 139.148, "state": "S (sleeping)", "threads": 106}, "restart_count": 244, "restart_count_per_hour": 13, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 245}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 237.851, "sample_count": 1338, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 353292, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:01:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.483, "last_frame_pushed_at": "2026-07-02T05:02:00+09:00", "last_frame_read_age_sec": 4.716, "last_frame_read_at": "2026-07-02T05:01:59+09:00", "mjpeg": {"age_sec": 4.466, "exists": true, "mtime": "2026-07-02T05:02:00.034517+09:00", "mtime_epoch": 1782936120.0345173, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165588}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:02:04+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1037.562, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 108, "pushed_frames": 108, "read_frames": 107, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.625, "pushed": 3.625, "read": 3.591}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.952, "count": 108, "max_ms": 16.154, "p95_ms": 10.914}, "gstreamer_push": {"avg_ms": 0.949, "count": 108, "max_ms": 2.824, "p95_ms": 1.844}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.38, "count": 108, "max_ms": 71.096, "p95_ms": 60.29}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.609, "count": 108, "max_ms": 91.71, "p95_ms": 83.579}, "yolo11_640_fire_11class": {"avg_ms": 53.024, "count": 108, "max_ms": 74.119, "p95_ms": 67.42}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.254, "count": 108, "max_ms": 91.501, "p95_ms": 84.944}}, "truncated_count": 0}, "processing_total": {"avg_ms": 226.553, "count": 108, "max_ms": 282.939, "p95_ms": 250.904}, "read_interval": {"avg_ms": 236.376, "count": 107, "max_ms": 293.52, "p95_ms": 260.732}, "triton_wait": {"avg_ms": 68.036, "count": 108, "max_ms": 91.722, "p95_ms": 87.358}, "visualize": {"avg_ms": 10.783, "count": 108, "max_ms": 14.99, "p95_ms": 12.568}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.795, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2947}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4383.367, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5566a9a4af80] error while decoding MB 37 40, bytestream -11", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 219, "h264_decode_error_count": 401, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:03:51+09:00", "last_error_summary": "[h264 @ 0x5566a9a4af80] error while decoding MB 37 40, bytestream -11", "last_exit_at": "2026-07-02T05:02:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.289, "last_started_at": "2026-07-02T05:02:02+09:00", "pid": 38992, "process": {"exists": false, "pid": 38992}, "restart_count": 244, "restart_count_per_hour": 13, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 245}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 246.197, "sample_count": 1284, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 353745, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:03:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-02T05:03:51+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-02T05:03:51+09:00", "mjpeg": {"age_sec": 0.048, "exists": true, "mtime": "2026-07-02T05:03:51.740379+09:00", "mtime_epoch": 1782936231.7403786, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169595}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:03:51+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1040.566, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 126, "pushed_frames": 126, "read_frames": 126, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.211, "pushed": 4.211, "read": 4.211}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.863, "count": 125, "max_ms": 13.496, "p95_ms": 9.839}, "gstreamer_push": {"avg_ms": 0.915, "count": 126, "max_ms": 2.361, "p95_ms": 0.942}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.054, "count": 125, "max_ms": 64.016, "p95_ms": 60.811}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.695, "count": 125, "max_ms": 92.651, "p95_ms": 87.461}, "yolo11_640_fire_11class": {"avg_ms": 53.16, "count": 125, "max_ms": 70.433, "p95_ms": 66.108}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.035, "count": 125, "max_ms": 92.663, "p95_ms": 86.289}}, "truncated_count": 0}, "processing_total": {"avg_ms": 229.468, "count": 126, "max_ms": 305.999, "p95_ms": 252.751}, "read_interval": {"avg_ms": 238.603, "count": 126, "max_ms": 318.374, "p95_ms": 262.143}, "triton_wait": {"avg_ms": 68.298, "count": 125, "max_ms": 92.674, "p95_ms": 87.828}, "visualize": {"avg_ms": 10.768, "count": 125, "max_ms": 14.326, "p95_ms": 12.403}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.918, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2947}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4382.297, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T05:03:51+09:00", "last_error_summary": "[h264 @ 0x5566a9a4af80] error while decoding MB 37 40, bytestream -11", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 219, "h264_decode_error_count": 401, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:03:51+09:00", "last_error_summary": "[h264 @ 0x5566a9a4af80] error while decoding MB 37 40, bytestream -11", "last_exit_at": "2026-07-02T05:03:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.474, "last_started_at": "2026-07-02T05:02:02+09:00", "pid": 38992, "process": {"exists": false, "pid": 38992}, "restart_count": 244, "restart_count_per_hour": 13, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 245}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 246.197, "sample_count": 1284, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 353746, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:03:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T05:03:51+09:00", "last_frame_read_age_sec": 0.238, "last_frame_read_at": "2026-07-02T05:03:51+09:00", "mjpeg": {"age_sec": 0.234, "exists": true, "mtime": "2026-07-02T05:03:51.740379+09:00", "mtime_epoch": 1782936231.7403786, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169595}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:03:51+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1040.566, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 126, "pushed_frames": 126, "read_frames": 125, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.216, "pushed": 4.216, "read": 4.183}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.869, "count": 125, "max_ms": 13.496, "p95_ms": 9.839}, "gstreamer_push": {"avg_ms": 0.915, "count": 126, "max_ms": 2.361, "p95_ms": 0.942}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.961, "count": 125, "max_ms": 64.016, "p95_ms": 60.811}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.667, "count": 125, "max_ms": 92.651, "p95_ms": 87.461}, "yolo11_640_fire_11class": {"avg_ms": 53.148, "count": 125, "max_ms": 70.433, "p95_ms": 66.108}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.015, "count": 125, "max_ms": 92.663, "p95_ms": 86.289}}, "truncated_count": 0}, "processing_total": {"avg_ms": 229.748, "count": 126, "max_ms": 305.999, "p95_ms": 252.751}, "read_interval": {"avg_ms": 238.85, "count": 125, "max_ms": 318.374, "p95_ms": 262.143}, "triton_wait": {"avg_ms": 68.279, "count": 125, "max_ms": 92.674, "p95_ms": 87.828}, "visualize": {"avg_ms": 10.775, "count": 125, "max_ms": 14.326, "p95_ms": 12.403}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.886, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2947}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4382.289, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 219, "h264_decode_error_count": 401, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:03:51+09:00", "last_error_summary": "[h264 @ 0x5566a9a4af80] error while decoding MB 37 40, bytestream -11", "last_exit_at": "2026-07-02T05:03:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.478, "last_started_at": "2026-07-02T05:02:02+09:00", "pid": 38992, "process": {"exists": false, "pid": 38992}, "restart_count": 244, "restart_count_per_hour": 13, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 245}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 246.197, "sample_count": 1284, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 353746, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:03:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T05:03:51+09:00", "last_frame_read_age_sec": 0.241, "last_frame_read_at": "2026-07-02T05:03:51+09:00", "mjpeg": {"age_sec": 0.237, "exists": true, "mtime": "2026-07-02T05:03:51.740379+09:00", "mtime_epoch": 1782936231.7403786, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169595}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:03:51+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1040.566, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 126, "pushed_frames": 126, "read_frames": 125, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.216, "pushed": 4.216, "read": 4.182}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.869, "count": 125, "max_ms": 13.496, "p95_ms": 9.839}, "gstreamer_push": {"avg_ms": 0.915, "count": 126, "max_ms": 2.361, "p95_ms": 0.942}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.961, "count": 125, "max_ms": 64.016, "p95_ms": 60.811}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.667, "count": 125, "max_ms": 92.651, "p95_ms": 87.461}, "yolo11_640_fire_11class": {"avg_ms": 53.148, "count": 125, "max_ms": 70.433, "p95_ms": 66.108}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.015, "count": 125, "max_ms": 92.663, "p95_ms": 86.289}}, "truncated_count": 0}, "processing_total": {"avg_ms": 229.748, "count": 126, "max_ms": 305.999, "p95_ms": 252.751}, "read_interval": {"avg_ms": 238.85, "count": 125, "max_ms": 318.374, "p95_ms": 262.143}, "triton_wait": {"avg_ms": 68.279, "count": 125, "max_ms": 92.674, "p95_ms": 87.828}, "visualize": {"avg_ms": 10.775, "count": 125, "max_ms": 14.326, "p95_ms": 12.403}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.889, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2947}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4492.816, "limit_mb": 8796093022207.996}, "detail": {"pid": 39099, "restart_count": 245, "start_count": 246}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 219, "h264_decode_error_count": 401, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:03:51+09:00", "last_error_summary": "[h264 @ 0x5566a9a4af80] error while decoding MB 37 40, bytestream -11", "last_exit_at": "2026-07-02T05:03:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T05:03:54+09:00", "pid": 39099, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.805, "name": "ffmpeg", "pid": 39099, "rss_mb": 141.805, "state": "S (sleeping)", "threads": 106}, "restart_count": 245, "restart_count_per_hour": 14, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 246}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 246.26, "sample_count": 1264, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 353746, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:03:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.08, "last_frame_pushed_at": "2026-07-02T05:03:51+09:00", "last_frame_read_age_sec": 4.315, "last_frame_read_at": "2026-07-02T05:03:51+09:00", "mjpeg": {"age_sec": 4.067, "exists": true, "mtime": "2026-07-02T05:03:51.984383+09:00", "mtime_epoch": 1782936231.9843826, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169688}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:03:56+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1040.566, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 108, "pushed_frames": 108, "read_frames": 107, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.613, "pushed": 3.613, "read": 3.58}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.933, "count": 107, "max_ms": 13.496, "p95_ms": 9.839}, "gstreamer_push": {"avg_ms": 0.924, "count": 108, "max_ms": 2.361, "p95_ms": 0.966}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.688, "count": 107, "max_ms": 64.016, "p95_ms": 60.74}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.526, "count": 107, "max_ms": 92.651, "p95_ms": 87.528}, "yolo11_640_fire_11class": {"avg_ms": 52.9, "count": 107, "max_ms": 68.696, "p95_ms": 65.528}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.834, "count": 107, "max_ms": 92.663, "p95_ms": 85.553}}, "truncated_count": 0}, "processing_total": {"avg_ms": 232.176, "count": 108, "max_ms": 305.999, "p95_ms": 252.781}, "read_interval": {"avg_ms": 241.206, "count": 107, "max_ms": 318.374, "p95_ms": 262.352}, "triton_wait": {"avg_ms": 68.129, "count": 107, "max_ms": 92.674, "p95_ms": 87.538}, "visualize": {"avg_ms": 10.818, "count": 107, "max_ms": 14.326, "p95_ms": 11.806}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.888, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2951}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4385.004, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x563946a65140] error while decoding MB 73 16, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 220, "h264_decode_error_count": 402, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:05:42+09:00", "last_error_summary": "[h264 @ 0x563946a65140] error while decoding MB 73 16, bytestream -5", "last_exit_at": "2026-07-02T05:03:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.241, "last_started_at": "2026-07-02T05:03:54+09:00", "pid": 39099, "process": {"exists": false, "pid": 39099}, "restart_count": 245, "restart_count_per_hour": 14, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 246}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 249.75, "sample_count": 1237, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 354199, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:05:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-02T05:05:42+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-02T05:05:42+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-02T05:05:42.250224+09:00", "mtime_epoch": 1782936342.250224, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165791}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:05:42+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1040.277, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 129, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.312, "pushed": 4.312, "read": 4.312}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.659, "count": 128, "max_ms": 11.611, "p95_ms": 9.341}, "gstreamer_push": {"avg_ms": 0.922, "count": 129, "max_ms": 2.86, "p95_ms": 1.071}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.391, "count": 128, "max_ms": 67.552, "p95_ms": 60.09}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.924, "count": 128, "max_ms": 88.449, "p95_ms": 85.051}, "yolo11_640_fire_11class": {"avg_ms": 52.878, "count": 128, "max_ms": 69.592, "p95_ms": 66.245}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.804, "count": 128, "max_ms": 89.78, "p95_ms": 84.9}}, "truncated_count": 0}, "processing_total": {"avg_ms": 223.601, "count": 129, "max_ms": 282.124, "p95_ms": 247.511}, "read_interval": {"avg_ms": 233.315, "count": 129, "max_ms": 293.53, "p95_ms": 257.811}, "triton_wait": {"avg_ms": 67.63, "count": 128, "max_ms": 89.791, "p95_ms": 86.48}, "visualize": {"avg_ms": 10.558, "count": 128, "max_ms": 14.873, "p95_ms": 11.638}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.914, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2951}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4378.797, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T05:05:42+09:00", "last_error_summary": "[h264 @ 0x563946a65140] error while decoding MB 73 16, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 220, "h264_decode_error_count": 402, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:05:42+09:00", "last_error_summary": "[h264 @ 0x563946a65140] error while decoding MB 73 16, bytestream -5", "last_exit_at": "2026-07-02T05:05:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.403, "last_started_at": "2026-07-02T05:03:54+09:00", "pid": 39099, "process": {"exists": false, "pid": 39099}, "restart_count": 245, "restart_count_per_hour": 14, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 246}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 249.75, "sample_count": 1238, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 354200, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:05:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T05:05:42+09:00", "last_frame_read_age_sec": 0.213, "last_frame_read_at": "2026-07-02T05:05:42+09:00", "mjpeg": {"age_sec": 0.203, "exists": true, "mtime": "2026-07-02T05:05:42.250224+09:00", "mtime_epoch": 1782936342.250224, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165791}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:05:42+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1040.277, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 128, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.323, "pushed": 4.323, "read": 4.29}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.661, "count": 129, "max_ms": 11.611, "p95_ms": 9.341}, "gstreamer_push": {"avg_ms": 0.922, "count": 129, "max_ms": 2.86, "p95_ms": 1.071}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.454, "count": 129, "max_ms": 67.552, "p95_ms": 60.09}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.881, "count": 129, "max_ms": 88.449, "p95_ms": 85.051}, "yolo11_640_fire_11class": {"avg_ms": 52.886, "count": 129, "max_ms": 69.592, "p95_ms": 66.245}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.785, "count": 129, "max_ms": 89.78, "p95_ms": 84.9}}, "truncated_count": 0}, "processing_total": {"avg_ms": 223.401, "count": 129, "max_ms": 282.124, "p95_ms": 247.511}, "read_interval": {"avg_ms": 233.228, "count": 128, "max_ms": 293.53, "p95_ms": 257.811}, "triton_wait": {"avg_ms": 67.606, "count": 129, "max_ms": 89.791, "p95_ms": 86.48}, "visualize": {"avg_ms": 10.552, "count": 129, "max_ms": 14.873, "p95_ms": 11.638}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.84, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2951}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4378.789, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 220, "h264_decode_error_count": 402, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:05:42+09:00", "last_error_summary": "[h264 @ 0x563946a65140] error while decoding MB 73 16, bytestream -5", "last_exit_at": "2026-07-02T05:05:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.406, "last_started_at": "2026-07-02T05:03:54+09:00", "pid": 39099, "process": {"exists": false, "pid": 39099}, "restart_count": 245, "restart_count_per_hour": 14, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 246}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 249.75, "sample_count": 1238, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 354200, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:05:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T05:05:42+09:00", "last_frame_read_age_sec": 0.216, "last_frame_read_at": "2026-07-02T05:05:42+09:00", "mjpeg": {"age_sec": 0.206, "exists": true, "mtime": "2026-07-02T05:05:42.250224+09:00", "mtime_epoch": 1782936342.250224, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165791}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:05:42+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1040.277, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 128, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.323, "pushed": 4.323, "read": 4.289}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.661, "count": 129, "max_ms": 11.611, "p95_ms": 9.341}, "gstreamer_push": {"avg_ms": 0.922, "count": 129, "max_ms": 2.86, "p95_ms": 1.071}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.454, "count": 129, "max_ms": 67.552, "p95_ms": 60.09}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.881, "count": 129, "max_ms": 88.449, "p95_ms": 85.051}, "yolo11_640_fire_11class": {"avg_ms": 52.886, "count": 129, "max_ms": 69.592, "p95_ms": 66.245}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.785, "count": 129, "max_ms": 89.78, "p95_ms": 84.9}}, "truncated_count": 0}, "processing_total": {"avg_ms": 223.401, "count": 129, "max_ms": 282.124, "p95_ms": 247.511}, "read_interval": {"avg_ms": 233.228, "count": 128, "max_ms": 293.53, "p95_ms": 257.811}, "triton_wait": {"avg_ms": 67.606, "count": 129, "max_ms": 89.791, "p95_ms": 86.48}, "visualize": {"avg_ms": 10.552, "count": 129, "max_ms": 14.873, "p95_ms": 11.638}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.843, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2951}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4489.105, "limit_mb": 8796093022207.996}, "detail": {"pid": 39206, "restart_count": 246, "start_count": 247}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 220, "h264_decode_error_count": 402, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:05:42+09:00", "last_error_summary": "[h264 @ 0x563946a65140] error while decoding MB 73 16, bytestream -5", "last_exit_at": "2026-07-02T05:05:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T05:05:45+09:00", "pid": 39206, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.449, "name": "ffmpeg", "pid": 39206, "rss_mb": 136.449, "state": "S (sleeping)", "threads": 106}, "restart_count": 246, "restart_count_per_hour": 15, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 247}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 249.75, "sample_count": 1217, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 354200, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:05:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.838, "last_frame_pushed_at": "2026-07-02T05:05:42+09:00", "last_frame_read_age_sec": 5.047, "last_frame_read_at": "2026-07-02T05:05:42+09:00", "mjpeg": {"age_sec": 4.822, "exists": true, "mtime": "2026-07-02T05:05:42.466228+09:00", "mtime_epoch": 1782936342.4662275, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 166246}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:05:47+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1040.277, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 109, "pushed_frames": 109, "read_frames": 108, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.652, "pushed": 3.652, "read": 3.619}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.6, "count": 109, "max_ms": 11.611, "p95_ms": 9.465}, "gstreamer_push": {"avg_ms": 0.932, "count": 109, "max_ms": 2.86, "p95_ms": 1.366}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.341, "count": 109, "max_ms": 67.552, "p95_ms": 59.481}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.635, "count": 109, "max_ms": 88.449, "p95_ms": 82.683}, "yolo11_640_fire_11class": {"avg_ms": 52.855, "count": 109, "max_ms": 69.592, "p95_ms": 67.754}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.651, "count": 109, "max_ms": 89.78, "p95_ms": 85.745}}, "truncated_count": 0}, "processing_total": {"avg_ms": 221.892, "count": 109, "max_ms": 282.124, "p95_ms": 247.477}, "read_interval": {"avg_ms": 231.754, "count": 108, "max_ms": 293.53, "p95_ms": 256.679}, "triton_wait": {"avg_ms": 67.376, "count": 109, "max_ms": 89.791, "p95_ms": 86.48}, "visualize": {"avg_ms": 10.472, "count": 109, "max_ms": 14.873, "p95_ms": 11.638}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.845, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2956}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4383.031, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5613d24df640] error while decoding MB 1 30, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 221, "h264_decode_error_count": 403, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:07:50+09:00", "last_error_summary": "[h264 @ 0x5613d24df640] error while decoding MB 1 30, bytestream -5", "last_exit_at": "2026-07-02T05:05:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.392, "last_started_at": "2026-07-02T05:05:45+09:00", "pid": 39206, "process": {"exists": false, "pid": 39206}, "restart_count": 246, "restart_count_per_hour": 15, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 247}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 245.736, "sample_count": 1285, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 354771, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:07:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.045, "last_frame_pushed_at": "2026-07-02T05:07:50+09:00", "last_frame_read_age_sec": 0.037, "last_frame_read_at": "2026-07-02T05:07:50+09:00", "mjpeg": {"age_sec": 0.03, "exists": true, "mtime": "2026-07-02T05:07:50.648372+09:00", "mtime_epoch": 1782936470.6483722, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 159799}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:07:50+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1040.32, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 142, "pushed_frames": 142, "read_frames": 142, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.768, "pushed": 4.768, "read": 4.768}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.466, "count": 142, "max_ms": 13.304, "p95_ms": 8.68}, "gstreamer_push": {"avg_ms": 0.932, "count": 142, "max_ms": 2.964, "p95_ms": 1.069}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.755, "count": 142, "max_ms": 70.407, "p95_ms": 67.119}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.406, "count": 142, "max_ms": 91.519, "p95_ms": 86.688}, "yolo11_640_fire_11class": {"avg_ms": 54.874, "count": 142, "max_ms": 71.343, "p95_ms": 68.948}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.235, "count": 142, "max_ms": 90.519, "p95_ms": 86.564}}, "truncated_count": 0}, "processing_total": {"avg_ms": 201.639, "count": 142, "max_ms": 269.909, "p95_ms": 230.635}, "read_interval": {"avg_ms": 211.111, "count": 142, "max_ms": 280.343, "p95_ms": 241.876}, "triton_wait": {"avg_ms": 69.688, "count": 142, "max_ms": 91.53, "p95_ms": 87.586}, "visualize": {"avg_ms": 9.326, "count": 142, "max_ms": 14.076, "p95_ms": 11.246}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.784, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2956}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4382.324, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T05:07:50+09:00", "last_error_summary": "[h264 @ 0x5613d24df640] error while decoding MB 1 30, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 221, "h264_decode_error_count": 403, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:07:50+09:00", "last_error_summary": "[h264 @ 0x5613d24df640] error while decoding MB 1 30, bytestream -5", "last_exit_at": "2026-07-02T05:07:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.552, "last_started_at": "2026-07-02T05:05:45+09:00", "pid": 39206, "process": {"exists": false, "pid": 39206}, "restart_count": 246, "restart_count_per_hour": 15, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 247}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 245.736, "sample_count": 1285, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 354772, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:07:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T05:07:50+09:00", "last_frame_read_age_sec": 0.197, "last_frame_read_at": "2026-07-02T05:07:50+09:00", "mjpeg": {"age_sec": 0.191, "exists": true, "mtime": "2026-07-02T05:07:50.648372+09:00", "mtime_epoch": 1782936470.6483722, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 159799}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:07:50+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1040.32, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 143, "pushed_frames": 143, "read_frames": 142, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.776, "pushed": 4.776, "read": 4.742}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.462, "count": 142, "max_ms": 13.304, "p95_ms": 8.68}, "gstreamer_push": {"avg_ms": 0.932, "count": 143, "max_ms": 2.964, "p95_ms": 1.069}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.657, "count": 142, "max_ms": 70.407, "p95_ms": 67.036}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.291, "count": 142, "max_ms": 91.519, "p95_ms": 86.569}, "yolo11_640_fire_11class": {"avg_ms": 54.751, "count": 142, "max_ms": 70.761, "p95_ms": 68.912}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.088, "count": 142, "max_ms": 90.519, "p95_ms": 86.486}}, "truncated_count": 0}, "processing_total": {"avg_ms": 201.577, "count": 143, "max_ms": 269.909, "p95_ms": 230.635}, "read_interval": {"avg_ms": 211.111, "count": 142, "max_ms": 280.343, "p95_ms": 241.876}, "triton_wait": {"avg_ms": 69.567, "count": 142, "max_ms": 91.53, "p95_ms": 87.105}, "visualize": {"avg_ms": 9.321, "count": 142, "max_ms": 14.076, "p95_ms": 11.246}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.944, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2956}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4382.309, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 221, "h264_decode_error_count": 403, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:07:50+09:00", "last_error_summary": "[h264 @ 0x5613d24df640] error while decoding MB 1 30, bytestream -5", "last_exit_at": "2026-07-02T05:07:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.555, "last_started_at": "2026-07-02T05:05:45+09:00", "pid": 39206, "process": {"exists": false, "pid": 39206}, "restart_count": 246, "restart_count_per_hour": 15, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 247}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 245.736, "sample_count": 1285, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 354772, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:07:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T05:07:50+09:00", "last_frame_read_age_sec": 0.2, "last_frame_read_at": "2026-07-02T05:07:50+09:00", "mjpeg": {"age_sec": 0.194, "exists": true, "mtime": "2026-07-02T05:07:50.648372+09:00", "mtime_epoch": 1782936470.6483722, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 159799}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:07:50+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1040.32, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 143, "pushed_frames": 143, "read_frames": 142, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.775, "pushed": 4.775, "read": 4.742}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.462, "count": 142, "max_ms": 13.304, "p95_ms": 8.68}, "gstreamer_push": {"avg_ms": 0.932, "count": 143, "max_ms": 2.964, "p95_ms": 1.069}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.657, "count": 142, "max_ms": 70.407, "p95_ms": 67.036}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.291, "count": 142, "max_ms": 91.519, "p95_ms": 86.569}, "yolo11_640_fire_11class": {"avg_ms": 54.751, "count": 142, "max_ms": 70.761, "p95_ms": 68.912}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.088, "count": 142, "max_ms": 90.519, "p95_ms": 86.486}}, "truncated_count": 0}, "processing_total": {"avg_ms": 201.577, "count": 143, "max_ms": 269.909, "p95_ms": 230.635}, "read_interval": {"avg_ms": 211.111, "count": 142, "max_ms": 280.343, "p95_ms": 241.876}, "triton_wait": {"avg_ms": 69.567, "count": 142, "max_ms": 91.53, "p95_ms": 87.105}, "visualize": {"avg_ms": 9.321, "count": 142, "max_ms": 14.076, "p95_ms": 11.246}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.948, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2956}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4490.203, "limit_mb": 8796093022207.996}, "detail": {"pid": 39313, "restart_count": 247, "start_count": 248}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 221, "h264_decode_error_count": 403, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:07:50+09:00", "last_error_summary": "[h264 @ 0x5613d24df640] error while decoding MB 1 30, bytestream -5", "last_exit_at": "2026-07-02T05:07:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T05:07:53+09:00", "pid": 39313, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.02, "name": "ffmpeg", "pid": 39313, "rss_mb": 139.02, "state": "S (sleeping)", "threads": 106}, "restart_count": 247, "restart_count_per_hour": 16, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 248}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 245.679, "sample_count": 1264, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 354772, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:07:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.99, "last_frame_pushed_at": "2026-07-02T05:07:50+09:00", "last_frame_read_age_sec": 5.184, "last_frame_read_at": "2026-07-02T05:07:50+09:00", "mjpeg": {"age_sec": 4.977, "exists": true, "mtime": "2026-07-02T05:07:50.848376+09:00", "mtime_epoch": 1782936470.8483756, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 158399}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:07:55+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1040.32, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 121, "pushed_frames": 121, "read_frames": 120, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.034, "pushed": 4.034, "read": 4.001}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.385, "count": 120, "max_ms": 13.304, "p95_ms": 8.63}, "gstreamer_push": {"avg_ms": 0.935, "count": 121, "max_ms": 2.964, "p95_ms": 1.069}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.463, "count": 120, "max_ms": 70.407, "p95_ms": 67.036}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.808, "count": 120, "max_ms": 89.093, "p95_ms": 86.015}, "yolo11_640_fire_11class": {"avg_ms": 54.561, "count": 120, "max_ms": 70.761, "p95_ms": 68.686}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.745, "count": 120, "max_ms": 88.59, "p95_ms": 85.648}}, "truncated_count": 0}, "processing_total": {"avg_ms": 198.974, "count": 121, "max_ms": 269.909, "p95_ms": 228.603}, "read_interval": {"avg_ms": 208.507, "count": 120, "max_ms": 280.343, "p95_ms": 238.361}, "triton_wait": {"avg_ms": 69.125, "count": 120, "max_ms": 89.102, "p95_ms": 86.579}, "visualize": {"avg_ms": 9.188, "count": 120, "max_ms": 14.076, "p95_ms": 11.246}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2961}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4387.18, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56483f1ca8c0] error while decoding MB 77 7, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 222, "h264_decode_error_count": 404, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:10:02+09:00", "last_error_summary": "[h264 @ 0x56483f1ca8c0] error while decoding MB 77 7, bytestream -5", "last_exit_at": "2026-07-02T05:07:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.37, "last_started_at": "2026-07-02T05:07:53+09:00", "pid": 39313, "process": {"exists": false, "pid": 39313}, "restart_count": 247, "restart_count_per_hour": 16, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 248}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 235.72, "sample_count": 1329, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 355356, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:10:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-02T05:10:02+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-02T05:10:02+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-02T05:10:02.150571+09:00", "mtime_epoch": 1782936602.1505709, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172676}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:10:02+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1043.453, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 135, "pushed_frames": 135, "read_frames": 135, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.506, "pushed": 4.506, "read": 4.506}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.911, "count": 134, "max_ms": 14.275, "p95_ms": 8.715}, "gstreamer_push": {"avg_ms": 0.952, "count": 135, "max_ms": 2.935, "p95_ms": 1.858}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.294, "count": 134, "max_ms": 73.25, "p95_ms": 69.535}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.494, "count": 134, "max_ms": 90.692, "p95_ms": 89.033}, "yolo11_640_fire_11class": {"avg_ms": 55.036, "count": 134, "max_ms": 71.323, "p95_ms": 69.555}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.885, "count": 134, "max_ms": 90.011, "p95_ms": 88.037}}, "truncated_count": 0}, "processing_total": {"avg_ms": 213.759, "count": 135, "max_ms": 281.481, "p95_ms": 236.644}, "read_interval": {"avg_ms": 223.23, "count": 135, "max_ms": 290.574, "p95_ms": 246.284}, "triton_wait": {"avg_ms": 70.806, "count": 134, "max_ms": 90.703, "p95_ms": 89.786}, "visualize": {"avg_ms": 9.698, "count": 134, "max_ms": 11.19, "p95_ms": 10.502}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.963, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2961}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4386.355, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T05:10:02+09:00", "last_error_summary": "[h264 @ 0x56483f1ca8c0] error while decoding MB 77 7, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 222, "h264_decode_error_count": 404, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:10:02+09:00", "last_error_summary": "[h264 @ 0x56483f1ca8c0] error while decoding MB 77 7, bytestream -5", "last_exit_at": "2026-07-02T05:10:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.533, "last_started_at": "2026-07-02T05:07:53+09:00", "pid": 39313, "process": {"exists": false, "pid": 39313}, "restart_count": 247, "restart_count_per_hour": 16, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 248}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 235.72, "sample_count": 1329, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 355357, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:10:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T05:10:02+09:00", "last_frame_read_age_sec": 0.215, "last_frame_read_at": "2026-07-02T05:10:02+09:00", "mjpeg": {"age_sec": 0.207, "exists": true, "mtime": "2026-07-02T05:10:02.150571+09:00", "mtime_epoch": 1782936602.1505709, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172676}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:10:02+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1043.453, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 135, "pushed_frames": 135, "read_frames": 134, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.513, "pushed": 4.513, "read": 4.48}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.912, "count": 134, "max_ms": 14.275, "p95_ms": 8.715}, "gstreamer_push": {"avg_ms": 0.937, "count": 135, "max_ms": 2.935, "p95_ms": 1.808}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.266, "count": 134, "max_ms": 73.25, "p95_ms": 69.535}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.539, "count": 134, "max_ms": 90.692, "p95_ms": 89.033}, "yolo11_640_fire_11class": {"avg_ms": 55.077, "count": 134, "max_ms": 71.323, "p95_ms": 69.555}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.923, "count": 134, "max_ms": 90.011, "p95_ms": 88.037}}, "truncated_count": 0}, "processing_total": {"avg_ms": 213.653, "count": 135, "max_ms": 281.481, "p95_ms": 236.644}, "read_interval": {"avg_ms": 223.129, "count": 134, "max_ms": 290.574, "p95_ms": 246.284}, "triton_wait": {"avg_ms": 70.85, "count": 134, "max_ms": 90.703, "p95_ms": 89.786}, "visualize": {"avg_ms": 9.698, "count": 134, "max_ms": 11.19, "p95_ms": 10.502}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.911, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2961}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4386.422, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 222, "h264_decode_error_count": 404, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:10:02+09:00", "last_error_summary": "[h264 @ 0x56483f1ca8c0] error while decoding MB 77 7, bytestream -5", "last_exit_at": "2026-07-02T05:10:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.536, "last_started_at": "2026-07-02T05:07:53+09:00", "pid": 39313, "process": {"exists": false, "pid": 39313}, "restart_count": 247, "restart_count_per_hour": 16, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 248}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 235.72, "sample_count": 1329, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 355357, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:10:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T05:10:02+09:00", "last_frame_read_age_sec": 0.219, "last_frame_read_at": "2026-07-02T05:10:02+09:00", "mjpeg": {"age_sec": 0.211, "exists": true, "mtime": "2026-07-02T05:10:02.150571+09:00", "mtime_epoch": 1782936602.1505709, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172676}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:10:02+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1043.453, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 135, "pushed_frames": 135, "read_frames": 134, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.513, "pushed": 4.513, "read": 4.479}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.912, "count": 134, "max_ms": 14.275, "p95_ms": 8.715}, "gstreamer_push": {"avg_ms": 0.937, "count": 135, "max_ms": 2.935, "p95_ms": 1.808}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.266, "count": 134, "max_ms": 73.25, "p95_ms": 69.535}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.539, "count": 134, "max_ms": 90.692, "p95_ms": 89.033}, "yolo11_640_fire_11class": {"avg_ms": 55.077, "count": 134, "max_ms": 71.323, "p95_ms": 69.555}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.923, "count": 134, "max_ms": 90.011, "p95_ms": 88.037}}, "truncated_count": 0}, "processing_total": {"avg_ms": 213.653, "count": 135, "max_ms": 281.481, "p95_ms": 236.644}, "read_interval": {"avg_ms": 223.129, "count": 134, "max_ms": 290.574, "p95_ms": 246.284}, "triton_wait": {"avg_ms": 70.85, "count": 134, "max_ms": 90.703, "p95_ms": 89.786}, "visualize": {"avg_ms": 9.698, "count": 134, "max_ms": 11.19, "p95_ms": 10.502}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.914, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2961}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4499.836, "limit_mb": 8796093022207.996}, "detail": {"pid": 39420, "restart_count": 248, "start_count": 249}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 222, "h264_decode_error_count": 404, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:10:02+09:00", "last_error_summary": "[h264 @ 0x56483f1ca8c0] error while decoding MB 77 7, bytestream -5", "last_exit_at": "2026-07-02T05:10:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T05:10:04+09:00", "pid": 39420, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.516, "name": "ffmpeg", "pid": 39420, "rss_mb": 141.516, "state": "S (sleeping)", "threads": 106}, "restart_count": 248, "restart_count_per_hour": 17, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 249}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 235.72, "sample_count": 1310, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 355357, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:10:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.461, "last_frame_pushed_at": "2026-07-02T05:10:02+09:00", "last_frame_read_age_sec": 4.674, "last_frame_read_at": "2026-07-02T05:10:02+09:00", "mjpeg": {"age_sec": 4.446, "exists": true, "mtime": "2026-07-02T05:10:02.370574+09:00", "mtime_epoch": 1782936602.3705745, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165660}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:10:06+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1043.453, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 115, "pushed_frames": 115, "read_frames": 114, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.858, "pushed": 3.858, "read": 3.824}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.89, "count": 115, "max_ms": 14.275, "p95_ms": 8.538}, "gstreamer_push": {"avg_ms": 0.949, "count": 115, "max_ms": 2.935, "p95_ms": 1.858}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.1, "count": 115, "max_ms": 73.25, "p95_ms": 69.535}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.458, "count": 115, "max_ms": 90.692, "p95_ms": 89.137}, "yolo11_640_fire_11class": {"avg_ms": 55.005, "count": 115, "max_ms": 71.323, "p95_ms": 69.555}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.869, "count": 115, "max_ms": 90.011, "p95_ms": 87.916}}, "truncated_count": 0}, "processing_total": {"avg_ms": 212.94, "count": 115, "max_ms": 276.658, "p95_ms": 236.016}, "read_interval": {"avg_ms": 222.383, "count": 114, "max_ms": 286.341, "p95_ms": 246.284}, "triton_wait": {"avg_ms": 70.747, "count": 115, "max_ms": 90.703, "p95_ms": 90.015}, "visualize": {"avg_ms": 9.676, "count": 115, "max_ms": 11.19, "p95_ms": 10.416}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.811, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2965}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4382.703, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x560e64a51b00] error while decoding MB 24 11, bytestream -14", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 223, "h264_decode_error_count": 405, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:11:52+09:00", "last_error_summary": "[h264 @ 0x560e64a51b00] error while decoding MB 24 11, bytestream -14", "last_exit_at": "2026-07-02T05:10:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.735, "last_started_at": "2026-07-02T05:10:04+09:00", "pid": 39420, "process": {"exists": false, "pid": 39420}, "restart_count": 248, "restart_count_per_hour": 16, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 249}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 234.543, "sample_count": 1337, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 355835, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:11:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-02T05:11:52+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-02T05:11:52+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-02T05:11:52.504416+09:00", "mtime_epoch": 1782936712.5044165, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172050}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:11:52+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1038.473, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 138, "pushed_frames": 138, "read_frames": 138, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.603, "pushed": 4.603, "read": 4.603}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.876, "count": 137, "max_ms": 11.394, "p95_ms": 8.786}, "gstreamer_push": {"avg_ms": 0.963, "count": 138, "max_ms": 3.131, "p95_ms": 1.833}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.895, "count": 137, "max_ms": 72.998, "p95_ms": 65.556}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.772, "count": 137, "max_ms": 92.12, "p95_ms": 89.586}, "yolo11_640_fire_11class": {"avg_ms": 54.052, "count": 137, "max_ms": 72.472, "p95_ms": 68.658}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.976, "count": 137, "max_ms": 91.801, "p95_ms": 88.194}}, "truncated_count": 0}, "processing_total": {"avg_ms": 210.022, "count": 138, "max_ms": 276.999, "p95_ms": 232.544}, "read_interval": {"avg_ms": 218.494, "count": 138, "max_ms": 287.803, "p95_ms": 242.208}, "triton_wait": {"avg_ms": 69.962, "count": 137, "max_ms": 92.133, "p95_ms": 89.787}, "visualize": {"avg_ms": 9.739, "count": 137, "max_ms": 12.377, "p95_ms": 11.034}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.978, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2965}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4381.871, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T05:11:52+09:00", "last_error_summary": "[h264 @ 0x560e64a51b00] error while decoding MB 24 11, bytestream -14", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 223, "h264_decode_error_count": 405, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:11:52+09:00", "last_error_summary": "[h264 @ 0x560e64a51b00] error while decoding MB 24 11, bytestream -14", "last_exit_at": "2026-07-02T05:11:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.913, "last_started_at": "2026-07-02T05:10:04+09:00", "pid": 39420, "process": {"exists": false, "pid": 39420}, "restart_count": 248, "restart_count_per_hour": 16, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 249}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 234.543, "sample_count": 1337, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 355836, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:11:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T05:11:52+09:00", "last_frame_read_age_sec": 0.232, "last_frame_read_at": "2026-07-02T05:11:52+09:00", "mjpeg": {"age_sec": 0.224, "exists": true, "mtime": "2026-07-02T05:11:52.504416+09:00", "mtime_epoch": 1782936712.5044165, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172050}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:11:52+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1038.473, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 138, "pushed_frames": 138, "read_frames": 137, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.61, "pushed": 4.61, "read": 4.577}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.877, "count": 137, "max_ms": 11.394, "p95_ms": 8.786}, "gstreamer_push": {"avg_ms": 0.963, "count": 138, "max_ms": 3.131, "p95_ms": 1.833}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.961, "count": 137, "max_ms": 72.998, "p95_ms": 65.556}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.886, "count": 137, "max_ms": 92.12, "p95_ms": 89.586}, "yolo11_640_fire_11class": {"avg_ms": 54.151, "count": 137, "max_ms": 72.472, "p95_ms": 68.658}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.101, "count": 137, "max_ms": 91.801, "p95_ms": 88.194}}, "truncated_count": 0}, "processing_total": {"avg_ms": 210.019, "count": 138, "max_ms": 276.999, "p95_ms": 232.544}, "read_interval": {"avg_ms": 218.345, "count": 137, "max_ms": 287.803, "p95_ms": 242.208}, "triton_wait": {"avg_ms": 70.076, "count": 137, "max_ms": 92.133, "p95_ms": 89.787}, "visualize": {"avg_ms": 9.74, "count": 137, "max_ms": 12.377, "p95_ms": 11.034}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.935, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2965}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4381.863, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 223, "h264_decode_error_count": 405, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:11:52+09:00", "last_error_summary": "[h264 @ 0x560e64a51b00] error while decoding MB 24 11, bytestream -14", "last_exit_at": "2026-07-02T05:11:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.917, "last_started_at": "2026-07-02T05:10:04+09:00", "pid": 39420, "process": {"exists": false, "pid": 39420}, "restart_count": 248, "restart_count_per_hour": 16, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 249}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 234.543, "sample_count": 1337, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 355836, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:11:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T05:11:52+09:00", "last_frame_read_age_sec": 0.235, "last_frame_read_at": "2026-07-02T05:11:52+09:00", "mjpeg": {"age_sec": 0.228, "exists": true, "mtime": "2026-07-02T05:11:52.504416+09:00", "mtime_epoch": 1782936712.5044165, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172050}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:11:52+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1038.473, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 138, "pushed_frames": 138, "read_frames": 137, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.61, "pushed": 4.61, "read": 4.576}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.877, "count": 137, "max_ms": 11.394, "p95_ms": 8.786}, "gstreamer_push": {"avg_ms": 0.963, "count": 138, "max_ms": 3.131, "p95_ms": 1.833}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.961, "count": 137, "max_ms": 72.998, "p95_ms": 65.556}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.886, "count": 137, "max_ms": 92.12, "p95_ms": 89.586}, "yolo11_640_fire_11class": {"avg_ms": 54.151, "count": 137, "max_ms": 72.472, "p95_ms": 68.658}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.101, "count": 137, "max_ms": 91.801, "p95_ms": 88.194}}, "truncated_count": 0}, "processing_total": {"avg_ms": 210.019, "count": 138, "max_ms": 276.999, "p95_ms": 232.544}, "read_interval": {"avg_ms": 218.345, "count": 137, "max_ms": 287.803, "p95_ms": 242.208}, "triton_wait": {"avg_ms": 70.076, "count": 137, "max_ms": 92.133, "p95_ms": 89.787}, "visualize": {"avg_ms": 9.74, "count": 137, "max_ms": 12.377, "p95_ms": 11.034}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.938, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2965}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4495.09, "limit_mb": 8796093022207.996}, "detail": {"pid": 39527, "restart_count": 249, "start_count": 250}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 223, "h264_decode_error_count": 405, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:11:52+09:00", "last_error_summary": "[h264 @ 0x560e64a51b00] error while decoding MB 24 11, bytestream -14", "last_exit_at": "2026-07-02T05:11:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T05:11:55+09:00", "pid": 39527, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.492, "name": "ffmpeg", "pid": 39527, "rss_mb": 141.492, "state": "S (sleeping)", "threads": 106}, "restart_count": 249, "restart_count_per_hour": 17, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 250}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 234.543, "sample_count": 1315, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 355836, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:11:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.644, "last_frame_pushed_at": "2026-07-02T05:11:52+09:00", "last_frame_read_age_sec": 4.873, "last_frame_read_at": "2026-07-02T05:11:52+09:00", "mjpeg": {"age_sec": 4.63, "exists": true, "mtime": "2026-07-02T05:11:52.740420+09:00", "mtime_epoch": 1782936712.7404203, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171989}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:11:57+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1038.473, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.918, "pushed": 3.918, "read": 3.884}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.872, "count": 117, "max_ms": 11.394, "p95_ms": 8.786}, "gstreamer_push": {"avg_ms": 0.978, "count": 117, "max_ms": 3.131, "p95_ms": 1.851}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.475, "count": 117, "max_ms": 72.998, "p95_ms": 65.556}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.476, "count": 117, "max_ms": 92.12, "p95_ms": 89.093}, "yolo11_640_fire_11class": {"avg_ms": 53.986, "count": 117, "max_ms": 72.472, "p95_ms": 68.658}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.799, "count": 117, "max_ms": 91.801, "p95_ms": 88.384}}, "truncated_count": 0}, "processing_total": {"avg_ms": 208.928, "count": 117, "max_ms": 276.999, "p95_ms": 232.515}, "read_interval": {"avg_ms": 217.321, "count": 116, "max_ms": 287.803, "p95_ms": 242.208}, "triton_wait": {"avg_ms": 69.659, "count": 117, "max_ms": 92.133, "p95_ms": 89.596}, "visualize": {"avg_ms": 9.695, "count": 117, "max_ms": 12.377, "p95_ms": 11.062}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.864, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2969}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4387.898, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T05:11:52+09:00", "last_error_summary": "[h264 @ 0x560e64a51b00] error while decoding MB 24 11, bytestream -14", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 223, "h264_decode_error_count": 405, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:11:52+09:00", "last_error_summary": "[h264 @ 0x560e64a51b00] error while decoding MB 24 11, bytestream -14", "last_exit_at": "2026-07-02T05:13:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.983, "last_started_at": "2026-07-02T05:11:55+09:00", "pid": 39527, "process": {"exists": false, "pid": 39527}, "restart_count": 249, "restart_count_per_hour": 17, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 250}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 236.644, "sample_count": 1307, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 356309, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:13:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T05:13:44+09:00", "last_frame_read_age_sec": 0.212, "last_frame_read_at": "2026-07-02T05:13:44+09:00", "mjpeg": {"age_sec": 0.202, "exists": true, "mtime": "2026-07-02T05:13:44.150286+09:00", "mtime_epoch": 1782936824.1502857, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169055}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:13:44+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1041.242, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 134, "pushed_frames": 134, "read_frames": 133, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.479, "pushed": 4.479, "read": 4.445}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.909, "count": 133, "max_ms": 10.139, "p95_ms": 8.336}, "gstreamer_push": {"avg_ms": 0.906, "count": 134, "max_ms": 2.779, "p95_ms": 0.98}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.175, "count": 133, "max_ms": 71.189, "p95_ms": 65.422}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.952, "count": 133, "max_ms": 92.493, "p95_ms": 89.052}, "yolo11_640_fire_11class": {"avg_ms": 54.532, "count": 133, "max_ms": 76.757, "p95_ms": 69.609}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.197, "count": 133, "max_ms": 91.42, "p95_ms": 88.179}}, "truncated_count": 0}, "processing_total": {"avg_ms": 215.213, "count": 134, "max_ms": 287.79, "p95_ms": 236.315}, "read_interval": {"avg_ms": 225.04, "count": 133, "max_ms": 302.209, "p95_ms": 246.643}, "triton_wait": {"avg_ms": 70.158, "count": 133, "max_ms": 92.503, "p95_ms": 89.061}, "visualize": {"avg_ms": 9.834, "count": 133, "max_ms": 11.477, "p95_ms": 10.409}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.921, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2969}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4387.996, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 223, "h264_decode_error_count": 405, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:11:52+09:00", "last_error_summary": "[h264 @ 0x560e64a51b00] error while decoding MB 24 11, bytestream -14", "last_exit_at": "2026-07-02T05:13:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.986, "last_started_at": "2026-07-02T05:11:55+09:00", "pid": 39527, "process": {"exists": false, "pid": 39527}, "restart_count": 249, "restart_count_per_hour": 17, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 250}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 236.644, "sample_count": 1307, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 356309, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:13:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T05:13:44+09:00", "last_frame_read_age_sec": 0.216, "last_frame_read_at": "2026-07-02T05:13:44+09:00", "mjpeg": {"age_sec": 0.205, "exists": true, "mtime": "2026-07-02T05:13:44.150286+09:00", "mtime_epoch": 1782936824.1502857, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169055}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:13:44+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1041.242, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 134, "pushed_frames": 134, "read_frames": 133, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.478, "pushed": 4.478, "read": 4.445}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.909, "count": 133, "max_ms": 10.139, "p95_ms": 8.336}, "gstreamer_push": {"avg_ms": 0.906, "count": 134, "max_ms": 2.779, "p95_ms": 0.98}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.175, "count": 133, "max_ms": 71.189, "p95_ms": 65.422}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.952, "count": 133, "max_ms": 92.493, "p95_ms": 89.052}, "yolo11_640_fire_11class": {"avg_ms": 54.532, "count": 133, "max_ms": 76.757, "p95_ms": 69.609}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.197, "count": 133, "max_ms": 91.42, "p95_ms": 88.179}}, "truncated_count": 0}, "processing_total": {"avg_ms": 215.213, "count": 134, "max_ms": 287.79, "p95_ms": 236.315}, "read_interval": {"avg_ms": 225.04, "count": 133, "max_ms": 302.209, "p95_ms": 246.643}, "triton_wait": {"avg_ms": 70.158, "count": 133, "max_ms": 92.503, "p95_ms": 89.061}, "visualize": {"avg_ms": 9.834, "count": 133, "max_ms": 11.477, "p95_ms": 10.409}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.924, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2969}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4493.461, "limit_mb": 8796093022207.996}, "detail": {"pid": 39634, "restart_count": 250, "start_count": 251}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 223, "h264_decode_error_count": 405, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:11:52+09:00", "last_error_summary": "[h264 @ 0x560e64a51b00] error while decoding MB 24 11, bytestream -14", "last_exit_at": "2026-07-02T05:13:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T05:13:46+09:00", "pid": 39634, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.773, "name": "ffmpeg", "pid": 39634, "rss_mb": 138.773, "state": "S (sleeping)", "threads": 106}, "restart_count": 250, "restart_count_per_hour": 18, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 251}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 236.81, "sample_count": 1287, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 356309, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:13:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.441, "last_frame_pushed_at": "2026-07-02T05:13:44+09:00", "last_frame_read_age_sec": 4.651, "last_frame_read_at": "2026-07-02T05:13:44+09:00", "mjpeg": {"age_sec": 4.424, "exists": true, "mtime": "2026-07-02T05:13:44.366289+09:00", "mtime_epoch": 1782936824.3662894, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169127}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:13:48+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1041.242, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 114, "pushed_frames": 114, "read_frames": 113, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.814, "pushed": 3.814, "read": 3.78}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.926, "count": 113, "max_ms": 9.101, "p95_ms": 8.336}, "gstreamer_push": {"avg_ms": 0.909, "count": 114, "max_ms": 2.779, "p95_ms": 1.009}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.132, "count": 113, "max_ms": 71.189, "p95_ms": 65.422}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.977, "count": 113, "max_ms": 92.493, "p95_ms": 89.052}, "yolo11_640_fire_11class": {"avg_ms": 54.616, "count": 113, "max_ms": 76.757, "p95_ms": 70.651}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.138, "count": 113, "max_ms": 91.42, "p95_ms": 88.179}}, "truncated_count": 0}, "processing_total": {"avg_ms": 215.396, "count": 114, "max_ms": 287.79, "p95_ms": 236.315}, "read_interval": {"avg_ms": 225.25, "count": 113, "max_ms": 302.209, "p95_ms": 246.643}, "triton_wait": {"avg_ms": 70.2, "count": 113, "max_ms": 92.503, "p95_ms": 89.061}, "visualize": {"avg_ms": 9.815, "count": 113, "max_ms": 11.477, "p95_ms": 10.315}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.891, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2973}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4392.449, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55f6bf282200] error while decoding MB 9 18, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 224, "h264_decode_error_count": 406, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:15:35+09:00", "last_error_summary": "[h264 @ 0x55f6bf282200] error while decoding MB 9 18, bytestream -7", "last_exit_at": "2026-07-02T05:13:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.457, "last_started_at": "2026-07-02T05:13:46+09:00", "pid": 39634, "process": {"exists": false, "pid": 39634}, "restart_count": 250, "restart_count_per_hour": 18, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 251}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 245.531, "sample_count": 1282, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 356768, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:15:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.064, "last_frame_pushed_at": "2026-07-02T05:15:35+09:00", "last_frame_read_age_sec": 0.056, "last_frame_read_at": "2026-07-02T05:15:35+09:00", "mjpeg": {"age_sec": 0.05, "exists": true, "mtime": "2026-07-02T05:15:35.196147+09:00", "mtime_epoch": 1782936935.1961472, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 155378}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:15:35+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1044.328, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 127, "pushed_frames": 127, "read_frames": 127, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.24, "pushed": 4.24, "read": 4.24}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.612, "count": 126, "max_ms": 12.028, "p95_ms": 9.429}, "gstreamer_push": {"avg_ms": 0.94, "count": 127, "max_ms": 2.818, "p95_ms": 1.809}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.912, "count": 126, "max_ms": 70.674, "p95_ms": 59.691}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.241, "count": 126, "max_ms": 93.648, "p95_ms": 88.152}, "yolo11_640_fire_11class": {"avg_ms": 53.528, "count": 126, "max_ms": 72.205, "p95_ms": 68.204}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.371, "count": 126, "max_ms": 92.236, "p95_ms": 88.923}}, "truncated_count": 0}, "processing_total": {"avg_ms": 227.467, "count": 127, "max_ms": 296.753, "p95_ms": 258.996}, "read_interval": {"avg_ms": 237.209, "count": 127, "max_ms": 305.717, "p95_ms": 268.37}, "triton_wait": {"avg_ms": 70.529, "count": 126, "max_ms": 93.66, "p95_ms": 89.534}, "visualize": {"avg_ms": 10.413, "count": 126, "max_ms": 11.87, "p95_ms": 11.383}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.952, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2973}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4391.734, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T05:15:35+09:00", "last_error_summary": "[h264 @ 0x55f6bf282200] error while decoding MB 9 18, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 224, "h264_decode_error_count": 406, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:15:35+09:00", "last_error_summary": "[h264 @ 0x55f6bf282200] error while decoding MB 9 18, bytestream -7", "last_exit_at": "2026-07-02T05:15:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.614, "last_started_at": "2026-07-02T05:13:46+09:00", "pid": 39634, "process": {"exists": false, "pid": 39634}, "restart_count": 250, "restart_count_per_hour": 18, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 251}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 245.531, "sample_count": 1283, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 356769, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:15:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T05:15:35+09:00", "last_frame_read_age_sec": 0.213, "last_frame_read_at": "2026-07-02T05:15:35+09:00", "mjpeg": {"age_sec": 0.207, "exists": true, "mtime": "2026-07-02T05:15:35.196147+09:00", "mtime_epoch": 1782936935.1961472, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 155378}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:15:35+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1044.328, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 127, "pushed_frames": 127, "read_frames": 126, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.252, "pushed": 4.252, "read": 4.218}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.615, "count": 126, "max_ms": 12.028, "p95_ms": 9.429}, "gstreamer_push": {"avg_ms": 0.939, "count": 127, "max_ms": 2.818, "p95_ms": 1.809}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.837, "count": 126, "max_ms": 70.674, "p95_ms": 59.691}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.201, "count": 126, "max_ms": 93.648, "p95_ms": 88.152}, "yolo11_640_fire_11class": {"avg_ms": 53.446, "count": 126, "max_ms": 72.205, "p95_ms": 68.204}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.298, "count": 126, "max_ms": 92.236, "p95_ms": 88.923}}, "truncated_count": 0}, "processing_total": {"avg_ms": 227.298, "count": 127, "max_ms": 296.753, "p95_ms": 258.996}, "read_interval": {"avg_ms": 237.191, "count": 126, "max_ms": 305.717, "p95_ms": 268.37}, "triton_wait": {"avg_ms": 70.489, "count": 126, "max_ms": 93.66, "p95_ms": 89.534}, "visualize": {"avg_ms": 10.409, "count": 126, "max_ms": 11.87, "p95_ms": 11.383}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.871, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2973}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4391.703, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 224, "h264_decode_error_count": 406, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:15:35+09:00", "last_error_summary": "[h264 @ 0x55f6bf282200] error while decoding MB 9 18, bytestream -7", "last_exit_at": "2026-07-02T05:15:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.617, "last_started_at": "2026-07-02T05:13:46+09:00", "pid": 39634, "process": {"exists": false, "pid": 39634}, "restart_count": 250, "restart_count_per_hour": 18, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 251}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 245.531, "sample_count": 1282, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 356769, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:15:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T05:15:35+09:00", "last_frame_read_age_sec": 0.216, "last_frame_read_at": "2026-07-02T05:15:35+09:00", "mjpeg": {"age_sec": 0.21, "exists": true, "mtime": "2026-07-02T05:15:35.196147+09:00", "mtime_epoch": 1782936935.1961472, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 155378}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:15:35+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1044.328, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 127, "pushed_frames": 127, "read_frames": 126, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.251, "pushed": 4.251, "read": 4.218}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.615, "count": 126, "max_ms": 12.028, "p95_ms": 9.429}, "gstreamer_push": {"avg_ms": 0.939, "count": 127, "max_ms": 2.818, "p95_ms": 1.809}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.837, "count": 126, "max_ms": 70.674, "p95_ms": 59.691}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.201, "count": 126, "max_ms": 93.648, "p95_ms": 88.152}, "yolo11_640_fire_11class": {"avg_ms": 53.446, "count": 126, "max_ms": 72.205, "p95_ms": 68.204}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.298, "count": 126, "max_ms": 92.236, "p95_ms": 88.923}}, "truncated_count": 0}, "processing_total": {"avg_ms": 227.298, "count": 127, "max_ms": 296.753, "p95_ms": 258.996}, "read_interval": {"avg_ms": 237.191, "count": 126, "max_ms": 305.717, "p95_ms": 268.37}, "triton_wait": {"avg_ms": 70.489, "count": 126, "max_ms": 93.66, "p95_ms": 89.534}, "visualize": {"avg_ms": 10.409, "count": 126, "max_ms": 11.87, "p95_ms": 11.383}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.874, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2973}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4496.641, "limit_mb": 8796093022207.996}, "detail": {"pid": 39741, "restart_count": 251, "start_count": 252}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 224, "h264_decode_error_count": 406, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:15:35+09:00", "last_error_summary": "[h264 @ 0x55f6bf282200] error while decoding MB 9 18, bytestream -7", "last_exit_at": "2026-07-02T05:15:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T05:15:38+09:00", "pid": 39741, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.02, "name": "ffmpeg", "pid": 39741, "rss_mb": 139.02, "state": "S (sleeping)", "threads": 106}, "restart_count": 251, "restart_count_per_hour": 19, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 252}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 245.961, "sample_count": 1261, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 356769, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:15:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.882, "last_frame_pushed_at": "2026-07-02T05:15:35+09:00", "last_frame_read_age_sec": 5.092, "last_frame_read_at": "2026-07-02T05:15:35+09:00", "mjpeg": {"age_sec": 4.866, "exists": true, "mtime": "2026-07-02T05:15:35.416151+09:00", "mtime_epoch": 1782936935.4161508, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 156498}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:15:40+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1044.328, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 107, "pushed_frames": 107, "read_frames": 106, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.568, "pushed": 3.568, "read": 3.534}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.649, "count": 106, "max_ms": 12.028, "p95_ms": 9.498}, "gstreamer_push": {"avg_ms": 0.926, "count": 107, "max_ms": 1.943, "p95_ms": 1.743}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.797, "count": 106, "max_ms": 70.674, "p95_ms": 59.939}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.009, "count": 106, "max_ms": 93.648, "p95_ms": 89.038}, "yolo11_640_fire_11class": {"avg_ms": 53.793, "count": 106, "max_ms": 72.205, "p95_ms": 68.238}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.893, "count": 106, "max_ms": 92.236, "p95_ms": 89.97}}, "truncated_count": 0}, "processing_total": {"avg_ms": 227.312, "count": 107, "max_ms": 296.753, "p95_ms": 258.996}, "read_interval": {"avg_ms": 237.247, "count": 106, "max_ms": 305.717, "p95_ms": 268.37}, "triton_wait": {"avg_ms": 71.285, "count": 106, "max_ms": 93.66, "p95_ms": 89.983}, "visualize": {"avg_ms": 10.358, "count": 106, "max_ms": 11.87, "p95_ms": 11.339}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2977}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4390.781, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55ac5ca041c0] error while decoding MB 40 38, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 225, "h264_decode_error_count": 407, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:17:49+09:00", "last_error_summary": "[h264 @ 0x55ac5ca041c0] error while decoding MB 40 38, bytestream -5", "last_exit_at": "2026-07-02T05:15:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.692, "last_started_at": "2026-07-02T05:15:38+09:00", "pid": 39741, "process": {"exists": false, "pid": 39741}, "restart_count": 251, "restart_count_per_hour": 19, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 252}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 248.973, "sample_count": 1269, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 357338, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:17:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.048, "last_frame_pushed_at": "2026-07-02T05:17:49+09:00", "last_frame_read_age_sec": 0.04, "last_frame_read_at": "2026-07-02T05:17:49+09:00", "mjpeg": {"age_sec": 0.031, "exists": true, "mtime": "2026-07-02T05:17:49.942408+09:00", "mtime_epoch": 1782937069.9424076, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164317}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:17:49+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1044.008, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 131, "pushed_frames": 131, "read_frames": 131, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.381, "pushed": 4.381, "read": 4.381}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.24, "count": 130, "max_ms": 12.495, "p95_ms": 9.158}, "gstreamer_push": {"avg_ms": 0.983, "count": 131, "max_ms": 3.007, "p95_ms": 1.899}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.597, "count": 130, "max_ms": 72.446, "p95_ms": 65.601}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.631, "count": 130, "max_ms": 93.632, "p95_ms": 89.278}, "yolo11_640_fire_11class": {"avg_ms": 54.651, "count": 130, "max_ms": 72.433, "p95_ms": 69.595}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.283, "count": 130, "max_ms": 93.034, "p95_ms": 88.114}}, "truncated_count": 0}, "processing_total": {"avg_ms": 219.866, "count": 131, "max_ms": 306.572, "p95_ms": 262.671}, "read_interval": {"avg_ms": 229.664, "count": 131, "max_ms": 317.269, "p95_ms": 274.193}, "triton_wait": {"avg_ms": 69.94, "count": 130, "max_ms": 93.641, "p95_ms": 89.291}, "visualize": {"avg_ms": 10.002, "count": 130, "max_ms": 12.799, "p95_ms": 11.249}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.902, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2977}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4389.809, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T05:17:49+09:00", "last_error_summary": "[h264 @ 0x55ac5ca041c0] error while decoding MB 40 38, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 225, "h264_decode_error_count": 407, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:17:49+09:00", "last_error_summary": "[h264 @ 0x55ac5ca041c0] error while decoding MB 40 38, bytestream -5", "last_exit_at": "2026-07-02T05:17:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.873, "last_started_at": "2026-07-02T05:15:38+09:00", "pid": 39741, "process": {"exists": false, "pid": 39741}, "restart_count": 251, "restart_count_per_hour": 19, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 252}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 248.529, "sample_count": 1269, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 357339, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:17:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T05:17:50+09:00", "last_frame_read_age_sec": 0.221, "last_frame_read_at": "2026-07-02T05:17:49+09:00", "mjpeg": {"age_sec": 0.212, "exists": true, "mtime": "2026-07-02T05:17:49.942408+09:00", "mtime_epoch": 1782937069.9424076, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164317}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:17:50+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1044.008, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 131, "pushed_frames": 131, "read_frames": 130, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.386, "pushed": 4.386, "read": 4.353}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.242, "count": 131, "max_ms": 12.495, "p95_ms": 9.158}, "gstreamer_push": {"avg_ms": 0.983, "count": 131, "max_ms": 3.007, "p95_ms": 1.899}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.652, "count": 131, "max_ms": 72.446, "p95_ms": 65.601}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.634, "count": 131, "max_ms": 93.632, "p95_ms": 89.278}, "yolo11_640_fire_11class": {"avg_ms": 54.657, "count": 131, "max_ms": 72.433, "p95_ms": 69.595}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.278, "count": 131, "max_ms": 93.034, "p95_ms": 88.114}}, "truncated_count": 0}, "processing_total": {"avg_ms": 219.806, "count": 131, "max_ms": 306.572, "p95_ms": 262.671}, "read_interval": {"avg_ms": 229.622, "count": 130, "max_ms": 317.269, "p95_ms": 274.193}, "triton_wait": {"avg_ms": 69.941, "count": 131, "max_ms": 93.641, "p95_ms": 89.291}, "visualize": {"avg_ms": 10.007, "count": 131, "max_ms": 12.799, "p95_ms": 11.249}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.866, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2977}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4389.801, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 225, "h264_decode_error_count": 407, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:17:49+09:00", "last_error_summary": "[h264 @ 0x55ac5ca041c0] error while decoding MB 40 38, bytestream -5", "last_exit_at": "2026-07-02T05:17:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.876, "last_started_at": "2026-07-02T05:15:38+09:00", "pid": 39741, "process": {"exists": false, "pid": 39741}, "restart_count": 251, "restart_count_per_hour": 19, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 252}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 248.529, "sample_count": 1269, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 357339, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:17:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T05:17:50+09:00", "last_frame_read_age_sec": 0.224, "last_frame_read_at": "2026-07-02T05:17:49+09:00", "mjpeg": {"age_sec": 0.215, "exists": true, "mtime": "2026-07-02T05:17:49.942408+09:00", "mtime_epoch": 1782937069.9424076, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164317}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:17:50+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1044.008, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 131, "pushed_frames": 131, "read_frames": 130, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.386, "pushed": 4.386, "read": 4.352}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.242, "count": 131, "max_ms": 12.495, "p95_ms": 9.158}, "gstreamer_push": {"avg_ms": 0.983, "count": 131, "max_ms": 3.007, "p95_ms": 1.899}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.652, "count": 131, "max_ms": 72.446, "p95_ms": 65.601}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.634, "count": 131, "max_ms": 93.632, "p95_ms": 89.278}, "yolo11_640_fire_11class": {"avg_ms": 54.657, "count": 131, "max_ms": 72.433, "p95_ms": 69.595}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.278, "count": 131, "max_ms": 93.034, "p95_ms": 88.114}}, "truncated_count": 0}, "processing_total": {"avg_ms": 219.806, "count": 131, "max_ms": 306.572, "p95_ms": 262.671}, "read_interval": {"avg_ms": 229.622, "count": 130, "max_ms": 317.269, "p95_ms": 274.193}, "triton_wait": {"avg_ms": 69.941, "count": 131, "max_ms": 93.641, "p95_ms": 89.291}, "visualize": {"avg_ms": 10.007, "count": 131, "max_ms": 12.799, "p95_ms": 11.249}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.87, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2977}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4498.059, "limit_mb": 8796093022207.996}, "detail": {"pid": 39848, "restart_count": 252, "start_count": 253}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 225, "h264_decode_error_count": 407, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:17:49+09:00", "last_error_summary": "[h264 @ 0x55ac5ca041c0] error while decoding MB 40 38, bytestream -5", "last_exit_at": "2026-07-02T05:17:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T05:17:52+09:00", "pid": 39848, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.875, "name": "ffmpeg", "pid": 39848, "rss_mb": 138.875, "state": "S (sleeping)", "threads": 106}, "restart_count": 252, "restart_count_per_hour": 20, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 253}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 248.973, "sample_count": 1249, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 357339, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:17:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.65, "last_frame_pushed_at": "2026-07-02T05:17:50+09:00", "last_frame_read_age_sec": 4.869, "last_frame_read_at": "2026-07-02T05:17:49+09:00", "mjpeg": {"age_sec": 4.635, "exists": true, "mtime": "2026-07-02T05:17:50.166411+09:00", "mtime_epoch": 1782937070.1664114, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164716}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:17:54+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1044.008, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 110, "pushed_frames": 110, "read_frames": 109, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.689, "pushed": 3.689, "read": 3.656}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.246, "count": 110, "max_ms": 12.495, "p95_ms": 9.184}, "gstreamer_push": {"avg_ms": 0.977, "count": 110, "max_ms": 2.863, "p95_ms": 1.86}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.53, "count": 110, "max_ms": 72.446, "p95_ms": 65.601}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.731, "count": 110, "max_ms": 93.632, "p95_ms": 89.364}, "yolo11_640_fire_11class": {"avg_ms": 54.541, "count": 110, "max_ms": 72.433, "p95_ms": 69.595}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.302, "count": 110, "max_ms": 93.034, "p95_ms": 90.318}}, "truncated_count": 0}, "processing_total": {"avg_ms": 220.991, "count": 110, "max_ms": 306.572, "p95_ms": 263.3}, "read_interval": {"avg_ms": 230.844, "count": 109, "max_ms": 317.269, "p95_ms": 274.801}, "triton_wait": {"avg_ms": 70.038, "count": 110, "max_ms": 93.641, "p95_ms": 90.33}, "visualize": {"avg_ms": 10.042, "count": 110, "max_ms": 12.799, "p95_ms": 11.297}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.817, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2981}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4388.812, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55fd892935c0] error while decoding MB 59 28, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 226, "h264_decode_error_count": 408, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:19:41+09:00", "last_error_summary": "[h264 @ 0x55fd892935c0] error while decoding MB 59 28, bytestream -7", "last_exit_at": "2026-07-02T05:17:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.658, "last_started_at": "2026-07-02T05:17:52+09:00", "pid": 39848, "process": {"exists": false, "pid": 39848}, "restart_count": 252, "restart_count_per_hour": 19, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 253}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 247.928, "sample_count": 1263, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 357801, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:19:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-02T05:19:41+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-02T05:19:41+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-02T05:19:41.412279+09:00", "mtime_epoch": 1782937181.4122791, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 162475}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:19:41+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1039.199, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 130, "pushed_frames": 130, "read_frames": 130, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.34, "pushed": 4.34, "read": 4.34}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.448, "count": 129, "max_ms": 9.757, "p95_ms": 9.096}, "gstreamer_push": {"avg_ms": 0.933, "count": 130, "max_ms": 3.132, "p95_ms": 0.989}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.62, "count": 129, "max_ms": 66.775, "p95_ms": 62.123}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.597, "count": 129, "max_ms": 91.5, "p95_ms": 87.673}, "yolo11_640_fire_11class": {"avg_ms": 54.364, "count": 129, "max_ms": 70.499, "p95_ms": 68.596}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.971, "count": 129, "max_ms": 90.658, "p95_ms": 88.511}}, "truncated_count": 0}, "processing_total": {"avg_ms": 222.13, "count": 130, "max_ms": 313.767, "p95_ms": 246.779}, "read_interval": {"avg_ms": 231.808, "count": 130, "max_ms": 323.787, "p95_ms": 255.883}, "triton_wait": {"avg_ms": 70.167, "count": 129, "max_ms": 91.511, "p95_ms": 88.707}, "visualize": {"avg_ms": 10.249, "count": 129, "max_ms": 11.902, "p95_ms": 11.169}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.956, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2981}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4387.742, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T05:19:41+09:00", "last_error_summary": "[h264 @ 0x55fd892935c0] error while decoding MB 59 28, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 226, "h264_decode_error_count": 408, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:19:41+09:00", "last_error_summary": "[h264 @ 0x55fd892935c0] error while decoding MB 59 28, bytestream -7", "last_exit_at": "2026-07-02T05:19:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.874, "last_started_at": "2026-07-02T05:17:52+09:00", "pid": 39848, "process": {"exists": false, "pid": 39848}, "restart_count": 252, "restart_count_per_hour": 19, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 253}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 248.297, "sample_count": 1263, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 357802, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:19:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T05:19:41+09:00", "last_frame_read_age_sec": 0.269, "last_frame_read_at": "2026-07-02T05:19:41+09:00", "mjpeg": {"age_sec": 0.262, "exists": true, "mtime": "2026-07-02T05:19:41.412279+09:00", "mtime_epoch": 1782937181.4122791, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 162475}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:19:41+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1038.668, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 130, "pushed_frames": 130, "read_frames": 129, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.34, "pushed": 4.34, "read": 4.307}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.453, "count": 129, "max_ms": 9.757, "p95_ms": 9.096}, "gstreamer_push": {"avg_ms": 0.941, "count": 130, "max_ms": 3.132, "p95_ms": 1.818}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.63, "count": 129, "max_ms": 66.775, "p95_ms": 62.123}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.646, "count": 129, "max_ms": 91.5, "p95_ms": 87.673}, "yolo11_640_fire_11class": {"avg_ms": 54.413, "count": 129, "max_ms": 70.499, "p95_ms": 68.596}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.011, "count": 129, "max_ms": 90.658, "p95_ms": 88.511}}, "truncated_count": 0}, "processing_total": {"avg_ms": 222.361, "count": 130, "max_ms": 313.767, "p95_ms": 249.179}, "read_interval": {"avg_ms": 231.729, "count": 129, "max_ms": 323.787, "p95_ms": 255.883}, "triton_wait": {"avg_ms": 70.216, "count": 129, "max_ms": 91.511, "p95_ms": 88.707}, "visualize": {"avg_ms": 10.248, "count": 129, "max_ms": 11.902, "p95_ms": 11.169}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.954, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2981}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4387.668, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 226, "h264_decode_error_count": 408, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:19:41+09:00", "last_error_summary": "[h264 @ 0x55fd892935c0] error while decoding MB 59 28, bytestream -7", "last_exit_at": "2026-07-02T05:19:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.877, "last_started_at": "2026-07-02T05:17:52+09:00", "pid": 39848, "process": {"exists": false, "pid": 39848}, "restart_count": 252, "restart_count_per_hour": 19, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 253}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 248.297, "sample_count": 1263, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 357802, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:19:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T05:19:41+09:00", "last_frame_read_age_sec": 0.272, "last_frame_read_at": "2026-07-02T05:19:41+09:00", "mjpeg": {"age_sec": 0.265, "exists": true, "mtime": "2026-07-02T05:19:41.412279+09:00", "mtime_epoch": 1782937181.4122791, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 162475}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:19:41+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1038.668, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 130, "pushed_frames": 130, "read_frames": 129, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.339, "pushed": 4.339, "read": 4.306}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.453, "count": 129, "max_ms": 9.757, "p95_ms": 9.096}, "gstreamer_push": {"avg_ms": 0.941, "count": 130, "max_ms": 3.132, "p95_ms": 1.818}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.63, "count": 129, "max_ms": 66.775, "p95_ms": 62.123}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.646, "count": 129, "max_ms": 91.5, "p95_ms": 87.673}, "yolo11_640_fire_11class": {"avg_ms": 54.413, "count": 129, "max_ms": 70.499, "p95_ms": 68.596}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.011, "count": 129, "max_ms": 90.658, "p95_ms": 88.511}}, "truncated_count": 0}, "processing_total": {"avg_ms": 222.361, "count": 130, "max_ms": 313.767, "p95_ms": 249.179}, "read_interval": {"avg_ms": 231.729, "count": 129, "max_ms": 323.787, "p95_ms": 255.883}, "triton_wait": {"avg_ms": 70.216, "count": 129, "max_ms": 91.511, "p95_ms": 88.707}, "visualize": {"avg_ms": 10.248, "count": 129, "max_ms": 11.902, "p95_ms": 11.169}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.957, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2981}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4495.137, "limit_mb": 8796093022207.996}, "detail": {"pid": 39955, "restart_count": 253, "start_count": 254}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 226, "h264_decode_error_count": 408, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:19:41+09:00", "last_error_summary": "[h264 @ 0x55fd892935c0] error while decoding MB 59 28, bytestream -7", "last_exit_at": "2026-07-02T05:19:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T05:19:44+09:00", "pid": 39955, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.93, "name": "ffmpeg", "pid": 39955, "rss_mb": 138.93, "state": "S (sleeping)", "threads": 106}, "restart_count": 253, "restart_count_per_hour": 20, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 254}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 248.297, "sample_count": 1242, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 357802, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:19:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.802, "last_frame_pushed_at": "2026-07-02T05:19:41+09:00", "last_frame_read_age_sec": 5.067, "last_frame_read_at": "2026-07-02T05:19:41+09:00", "mjpeg": {"age_sec": 4.789, "exists": true, "mtime": "2026-07-02T05:19:41.684283+09:00", "mtime_epoch": 1782937181.6842835, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 162856}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:19:46+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1038.668, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 109, "pushed_frames": 109, "read_frames": 108, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.643, "pushed": 3.643, "read": 3.609}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.434, "count": 108, "max_ms": 9.757, "p95_ms": 9.057}, "gstreamer_push": {"avg_ms": 0.946, "count": 109, "max_ms": 3.132, "p95_ms": 1.828}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.0, "count": 108, "max_ms": 66.775, "p95_ms": 61.182}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.845, "count": 108, "max_ms": 91.5, "p95_ms": 88.694}, "yolo11_640_fire_11class": {"avg_ms": 54.23, "count": 108, "max_ms": 70.499, "p95_ms": 68.711}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.96, "count": 108, "max_ms": 90.658, "p95_ms": 89.671}}, "truncated_count": 0}, "processing_total": {"avg_ms": 223.055, "count": 109, "max_ms": 313.767, "p95_ms": 249.179}, "read_interval": {"avg_ms": 232.342, "count": 108, "max_ms": 323.787, "p95_ms": 255.883}, "triton_wait": {"avg_ms": 70.331, "count": 108, "max_ms": 91.511, "p95_ms": 89.682}, "visualize": {"avg_ms": 10.246, "count": 108, "max_ms": 11.902, "p95_ms": 11.169}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.924, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2985}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4392.586, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x557e9a208800] error while decoding MB 62 35, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 227, "h264_decode_error_count": 409, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:21:32+09:00", "last_error_summary": "[h264 @ 0x557e9a208800] error while decoding MB 62 35, bytestream -5", "last_exit_at": "2026-07-02T05:19:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.373, "last_started_at": "2026-07-02T05:19:44+09:00", "pid": 39955, "process": {"exists": false, "pid": 39955}, "restart_count": 253, "restart_count_per_hour": 20, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 254}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 247.248, "sample_count": 1260, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 358261, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:21:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-02T05:21:32+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-02T05:21:32+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-02T05:21:32.802150+09:00", "mtime_epoch": 1782937292.80215, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164078}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:21:32+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1042.371, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 132, "pushed_frames": 132, "read_frames": 132, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.406, "pushed": 4.406, "read": 4.406}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.407, "count": 131, "max_ms": 12.625, "p95_ms": 9.899}, "gstreamer_push": {"avg_ms": 0.928, "count": 132, "max_ms": 2.777, "p95_ms": 1.805}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.108, "count": 131, "max_ms": 67.827, "p95_ms": 60.565}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.757, "count": 131, "max_ms": 93.988, "p95_ms": 87.787}, "yolo11_640_fire_11class": {"avg_ms": 53.197, "count": 131, "max_ms": 72.356, "p95_ms": 68.861}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.507, "count": 131, "max_ms": 91.747, "p95_ms": 88.072}}, "truncated_count": 0}, "processing_total": {"avg_ms": 218.465, "count": 132, "max_ms": 301.514, "p95_ms": 240.294}, "read_interval": {"avg_ms": 228.159, "count": 132, "max_ms": 312.028, "p95_ms": 251.345}, "triton_wait": {"avg_ms": 69.138, "count": 131, "max_ms": 93.999, "p95_ms": 88.564}, "visualize": {"avg_ms": 10.066, "count": 131, "max_ms": 15.268, "p95_ms": 11.144}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.961, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2985}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4388.215, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T05:21:32+09:00", "last_error_summary": "[h264 @ 0x557e9a208800] error while decoding MB 62 35, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 227, "h264_decode_error_count": 409, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:21:32+09:00", "last_error_summary": "[h264 @ 0x557e9a208800] error while decoding MB 62 35, bytestream -5", "last_exit_at": "2026-07-02T05:21:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.555, "last_started_at": "2026-07-02T05:19:44+09:00", "pid": 39955, "process": {"exists": false, "pid": 39955}, "restart_count": 253, "restart_count_per_hour": 20, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 254}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 247.248, "sample_count": 1260, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 358262, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:21:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T05:21:33+09:00", "last_frame_read_age_sec": 0.233, "last_frame_read_at": "2026-07-02T05:21:32+09:00", "mjpeg": {"age_sec": 0.225, "exists": true, "mtime": "2026-07-02T05:21:32.802150+09:00", "mtime_epoch": 1782937292.80215, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164078}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:21:33+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1038.762, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 132, "pushed_frames": 132, "read_frames": 131, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.415, "pushed": 4.415, "read": 4.381}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.412, "count": 131, "max_ms": 12.625, "p95_ms": 9.899}, "gstreamer_push": {"avg_ms": 0.942, "count": 132, "max_ms": 2.816, "p95_ms": 1.837}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.086, "count": 131, "max_ms": 67.827, "p95_ms": 60.565}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.565, "count": 131, "max_ms": 93.988, "p95_ms": 87.787}, "yolo11_640_fire_11class": {"avg_ms": 53.132, "count": 131, "max_ms": 72.356, "p95_ms": 68.861}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.293, "count": 131, "max_ms": 91.747, "p95_ms": 88.072}}, "truncated_count": 0}, "processing_total": {"avg_ms": 218.609, "count": 132, "max_ms": 301.514, "p95_ms": 240.294}, "read_interval": {"avg_ms": 228.241, "count": 131, "max_ms": 312.028, "p95_ms": 251.345}, "triton_wait": {"avg_ms": 68.938, "count": 131, "max_ms": 93.999, "p95_ms": 88.564}, "visualize": {"avg_ms": 10.076, "count": 131, "max_ms": 15.268, "p95_ms": 11.22}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.901, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2985}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4388.207, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 227, "h264_decode_error_count": 409, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:21:32+09:00", "last_error_summary": "[h264 @ 0x557e9a208800] error while decoding MB 62 35, bytestream -5", "last_exit_at": "2026-07-02T05:21:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.558, "last_started_at": "2026-07-02T05:19:44+09:00", "pid": 39955, "process": {"exists": false, "pid": 39955}, "restart_count": 253, "restart_count_per_hour": 20, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 254}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 247.248, "sample_count": 1260, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 358262, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:21:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T05:21:33+09:00", "last_frame_read_age_sec": 0.236, "last_frame_read_at": "2026-07-02T05:21:32+09:00", "mjpeg": {"age_sec": 0.228, "exists": true, "mtime": "2026-07-02T05:21:32.802150+09:00", "mtime_epoch": 1782937292.80215, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164078}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:21:33+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1038.762, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 132, "pushed_frames": 132, "read_frames": 131, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.414, "pushed": 4.414, "read": 4.381}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.412, "count": 131, "max_ms": 12.625, "p95_ms": 9.899}, "gstreamer_push": {"avg_ms": 0.942, "count": 132, "max_ms": 2.816, "p95_ms": 1.837}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.086, "count": 131, "max_ms": 67.827, "p95_ms": 60.565}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.565, "count": 131, "max_ms": 93.988, "p95_ms": 87.787}, "yolo11_640_fire_11class": {"avg_ms": 53.132, "count": 131, "max_ms": 72.356, "p95_ms": 68.861}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.293, "count": 131, "max_ms": 91.747, "p95_ms": 88.072}}, "truncated_count": 0}, "processing_total": {"avg_ms": 218.609, "count": 132, "max_ms": 301.514, "p95_ms": 240.294}, "read_interval": {"avg_ms": 228.241, "count": 131, "max_ms": 312.028, "p95_ms": 251.345}, "triton_wait": {"avg_ms": 68.938, "count": 131, "max_ms": 93.999, "p95_ms": 88.564}, "visualize": {"avg_ms": 10.076, "count": 131, "max_ms": 15.268, "p95_ms": 11.22}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.904, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2985}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4493.5, "limit_mb": 8796093022207.996}, "detail": {"pid": 40062, "restart_count": 254, "start_count": 255}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 227, "h264_decode_error_count": 409, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:21:32+09:00", "last_error_summary": "[h264 @ 0x557e9a208800] error while decoding MB 62 35, bytestream -5", "last_exit_at": "2026-07-02T05:21:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T05:21:35+09:00", "pid": 40062, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.922, "name": "ffmpeg", "pid": 40062, "rss_mb": 138.922, "state": "S (sleeping)", "threads": 106}, "restart_count": 254, "restart_count_per_hour": 21, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 255}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 247.928, "sample_count": 1240, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 358262, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:21:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.435, "last_frame_pushed_at": "2026-07-02T05:21:33+09:00", "last_frame_read_age_sec": 4.666, "last_frame_read_at": "2026-07-02T05:21:32+09:00", "mjpeg": {"age_sec": 4.421, "exists": true, "mtime": "2026-07-02T05:21:33.038154+09:00", "mtime_epoch": 1782937293.038154, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164372}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:21:37+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1038.762, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 112, "pushed_frames": 112, "read_frames": 111, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.752, "pushed": 3.752, "read": 3.719}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.465, "count": 112, "max_ms": 12.598, "p95_ms": 9.899}, "gstreamer_push": {"avg_ms": 0.94, "count": 112, "max_ms": 2.816, "p95_ms": 1.837}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.033, "count": 112, "max_ms": 66.607, "p95_ms": 60.342}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.459, "count": 112, "max_ms": 93.988, "p95_ms": 87.787}, "yolo11_640_fire_11class": {"avg_ms": 52.95, "count": 112, "max_ms": 72.356, "p95_ms": 67.947}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.101, "count": 112, "max_ms": 91.747, "p95_ms": 87.799}}, "truncated_count": 0}, "processing_total": {"avg_ms": 219.043, "count": 112, "max_ms": 301.514, "p95_ms": 240.294}, "read_interval": {"avg_ms": 228.659, "count": 111, "max_ms": 312.028, "p95_ms": 251.345}, "triton_wait": {"avg_ms": 68.825, "count": 112, "max_ms": 93.999, "p95_ms": 87.997}, "visualize": {"avg_ms": 10.167, "count": 112, "max_ms": 15.268, "p95_ms": 11.293}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.85, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2989}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4396.023, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x558e270e66c0] error while decoding MB 1 33, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 228, "h264_decode_error_count": 410, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:23:24+09:00", "last_error_summary": "[h264 @ 0x558e270e66c0] error while decoding MB 1 33, bytestream -5", "last_exit_at": "2026-07-02T05:21:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.938, "last_started_at": "2026-07-02T05:21:35+09:00", "pid": 40062, "process": {"exists": false, "pid": 40062}, "restart_count": 254, "restart_count_per_hour": 20, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 255}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 250.792, "sample_count": 1244, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 358712, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:23:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-02T05:23:24+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-02T05:23:24+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-02T05:23:24.352024+09:00", "mtime_epoch": 1782937404.3520243, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 160365}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:23:24+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1045.004, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 126, "pushed_frames": 126, "read_frames": 126, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.201, "pushed": 4.201, "read": 4.201}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.013, "count": 125, "max_ms": 15.035, "p95_ms": 11.07}, "gstreamer_push": {"avg_ms": 0.9, "count": 126, "max_ms": 1.932, "p95_ms": 0.985}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.533, "count": 125, "max_ms": 66.377, "p95_ms": 60.138}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.259, "count": 125, "max_ms": 93.471, "p95_ms": 85.457}, "yolo11_640_fire_11class": {"avg_ms": 51.263, "count": 125, "max_ms": 73.472, "p95_ms": 67.026}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.032, "count": 125, "max_ms": 92.375, "p95_ms": 86.606}}, "truncated_count": 0}, "processing_total": {"avg_ms": 229.925, "count": 126, "max_ms": 305.223, "p95_ms": 258.926}, "read_interval": {"avg_ms": 239.66, "count": 126, "max_ms": 316.247, "p95_ms": 267.992}, "triton_wait": {"avg_ms": 66.7, "count": 125, "max_ms": 93.484, "p95_ms": 88.223}, "visualize": {"avg_ms": 10.723, "count": 125, "max_ms": 14.493, "p95_ms": 11.68}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.99, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2989}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4392.34, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T05:23:24+09:00", "last_error_summary": "[h264 @ 0x558e270e66c0] error while decoding MB 1 33, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 228, "h264_decode_error_count": 410, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:23:24+09:00", "last_error_summary": "[h264 @ 0x558e270e66c0] error while decoding MB 1 33, bytestream -5", "last_exit_at": "2026-07-02T05:23:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.12, "last_started_at": "2026-07-02T05:21:35+09:00", "pid": 40062, "process": {"exists": false, "pid": 40062}, "restart_count": 254, "restart_count_per_hour": 20, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 255}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 250.792, "sample_count": 1244, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 358713, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:23:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T05:23:24+09:00", "last_frame_read_age_sec": 0.233, "last_frame_read_at": "2026-07-02T05:23:24+09:00", "mjpeg": {"age_sec": 0.226, "exists": true, "mtime": "2026-07-02T05:23:24.352024+09:00", "mtime_epoch": 1782937404.3520243, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 160365}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:23:24+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1045.004, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 126, "pushed_frames": 126, "read_frames": 125, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.208, "pushed": 4.208, "read": 4.175}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.008, "count": 125, "max_ms": 15.035, "p95_ms": 11.07}, "gstreamer_push": {"avg_ms": 0.899, "count": 126, "max_ms": 1.932, "p95_ms": 0.985}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.733, "count": 125, "max_ms": 66.377, "p95_ms": 60.561}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.477, "count": 125, "max_ms": 93.471, "p95_ms": 87.467}, "yolo11_640_fire_11class": {"avg_ms": 51.488, "count": 125, "max_ms": 74.613, "p95_ms": 68.225}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.259, "count": 125, "max_ms": 92.375, "p95_ms": 88.036}}, "truncated_count": 0}, "processing_total": {"avg_ms": 229.685, "count": 126, "max_ms": 305.223, "p95_ms": 253.755}, "read_interval": {"avg_ms": 239.434, "count": 125, "max_ms": 316.247, "p95_ms": 264.181}, "triton_wait": {"avg_ms": 66.927, "count": 125, "max_ms": 93.484, "p95_ms": 88.68}, "visualize": {"avg_ms": 10.711, "count": 125, "max_ms": 14.493, "p95_ms": 11.68}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.94, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2989}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4392.332, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 228, "h264_decode_error_count": 410, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:23:24+09:00", "last_error_summary": "[h264 @ 0x558e270e66c0] error while decoding MB 1 33, bytestream -5", "last_exit_at": "2026-07-02T05:23:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.123, "last_started_at": "2026-07-02T05:21:35+09:00", "pid": 40062, "process": {"exists": false, "pid": 40062}, "restart_count": 254, "restart_count_per_hour": 20, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 255}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 250.792, "sample_count": 1244, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 358713, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:23:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T05:23:24+09:00", "last_frame_read_age_sec": 0.236, "last_frame_read_at": "2026-07-02T05:23:24+09:00", "mjpeg": {"age_sec": 0.23, "exists": true, "mtime": "2026-07-02T05:23:24.352024+09:00", "mtime_epoch": 1782937404.3520243, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 160365}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:23:24+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1045.004, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 126, "pushed_frames": 126, "read_frames": 125, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.208, "pushed": 4.208, "read": 4.175}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.008, "count": 125, "max_ms": 15.035, "p95_ms": 11.07}, "gstreamer_push": {"avg_ms": 0.899, "count": 126, "max_ms": 1.932, "p95_ms": 0.985}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.733, "count": 125, "max_ms": 66.377, "p95_ms": 60.561}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.477, "count": 125, "max_ms": 93.471, "p95_ms": 87.467}, "yolo11_640_fire_11class": {"avg_ms": 51.488, "count": 125, "max_ms": 74.613, "p95_ms": 68.225}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.259, "count": 125, "max_ms": 92.375, "p95_ms": 88.036}}, "truncated_count": 0}, "processing_total": {"avg_ms": 229.685, "count": 126, "max_ms": 305.223, "p95_ms": 253.755}, "read_interval": {"avg_ms": 239.434, "count": 125, "max_ms": 316.247, "p95_ms": 264.181}, "triton_wait": {"avg_ms": 66.927, "count": 125, "max_ms": 93.484, "p95_ms": 88.68}, "visualize": {"avg_ms": 10.711, "count": 125, "max_ms": 14.493, "p95_ms": 11.68}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.943, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2989}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4505.148, "limit_mb": 8796093022207.996}, "detail": {"pid": 40169, "restart_count": 255, "start_count": 256}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 228, "h264_decode_error_count": 410, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:23:24+09:00", "last_error_summary": "[h264 @ 0x558e270e66c0] error while decoding MB 1 33, bytestream -5", "last_exit_at": "2026-07-02T05:23:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T05:23:26+09:00", "pid": 40169, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.887, "name": "ffmpeg", "pid": 40169, "rss_mb": 138.887, "state": "S (sleeping)", "threads": 106}, "restart_count": 255, "restart_count_per_hour": 21, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 256}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 250.771, "sample_count": 1227, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 358713, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:23:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.135, "last_frame_pushed_at": "2026-07-02T05:23:24+09:00", "last_frame_read_age_sec": 4.365, "last_frame_read_at": "2026-07-02T05:23:24+09:00", "mjpeg": {"age_sec": 4.122, "exists": true, "mtime": "2026-07-02T05:23:24.588028+09:00", "mtime_epoch": 1782937404.5880284, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 160903}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:23:28+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1045.004, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 109, "pushed_frames": 109, "read_frames": 108, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.649, "pushed": 3.649, "read": 3.616}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.982, "count": 108, "max_ms": 12.145, "p95_ms": 10.93}, "gstreamer_push": {"avg_ms": 0.905, "count": 109, "max_ms": 1.932, "p95_ms": 1.13}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.27, "count": 108, "max_ms": 64.899, "p95_ms": 60.138}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.348, "count": 108, "max_ms": 93.179, "p95_ms": 87.467}, "yolo11_640_fire_11class": {"avg_ms": 51.31, "count": 108, "max_ms": 74.613, "p95_ms": 68.668}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.815, "count": 108, "max_ms": 90.701, "p95_ms": 86.606}}, "truncated_count": 0}, "processing_total": {"avg_ms": 228.565, "count": 109, "max_ms": 290.739, "p95_ms": 253.538}, "read_interval": {"avg_ms": 238.327, "count": 108, "max_ms": 300.823, "p95_ms": 262.888}, "triton_wait": {"avg_ms": 66.658, "count": 108, "max_ms": 93.194, "p95_ms": 88.68}, "visualize": {"avg_ms": 10.617, "count": 108, "max_ms": 12.071, "p95_ms": 11.319}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.87, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2993}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4388.602, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55fd42d68740] error while decoding MB 17 44, bytestream -46", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 229, "h264_decode_error_count": 411, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:25:18+09:00", "last_error_summary": "[h264 @ 0x55fd42d68740] error while decoding MB 17 44, bytestream -46", "last_exit_at": "2026-07-02T05:23:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.815, "last_started_at": "2026-07-02T05:23:26+09:00", "pid": 40169, "process": {"exists": false, "pid": 40169}, "restart_count": 255, "restart_count_per_hour": 21, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 256}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 253.753, "sample_count": 1234, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 359173, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:25:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.069, "last_frame_pushed_at": "2026-07-02T05:25:18+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-02T05:25:18+09:00", "mjpeg": {"age_sec": 0.052, "exists": true, "mtime": "2026-07-02T05:25:18.473943+09:00", "mtime_epoch": 1782937518.4739428, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165302}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:25:18+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1036.289, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 125, "pushed_frames": 125, "read_frames": 125, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.198, "pushed": 4.198, "read": 4.198}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.125, "count": 125, "max_ms": 14.522, "p95_ms": 11.468}, "gstreamer_push": {"avg_ms": 0.895, "count": 125, "max_ms": 2.366, "p95_ms": 0.938}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.903, "count": 125, "max_ms": 74.608, "p95_ms": 59.582}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.375, "count": 125, "max_ms": 89.767, "p95_ms": 86.382}, "yolo11_640_fire_11class": {"avg_ms": 52.104, "count": 125, "max_ms": 74.056, "p95_ms": 67.881}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.259, "count": 125, "max_ms": 93.577, "p95_ms": 88.25}}, "truncated_count": 0}, "processing_total": {"avg_ms": 230.008, "count": 125, "max_ms": 293.547, "p95_ms": 256.861}, "read_interval": {"avg_ms": 239.597, "count": 125, "max_ms": 302.563, "p95_ms": 265.648}, "triton_wait": {"avg_ms": 67.753, "count": 125, "max_ms": 93.587, "p95_ms": 88.31}, "visualize": {"avg_ms": 10.777, "count": 125, "max_ms": 13.042, "p95_ms": 11.934}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.778, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2993}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4391.434, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T05:25:18+09:00", "last_error_summary": "[h264 @ 0x55fd42d68740] error while decoding MB 17 44, bytestream -46", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 229, "h264_decode_error_count": 411, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:25:18+09:00", "last_error_summary": "[h264 @ 0x55fd42d68740] error while decoding MB 17 44, bytestream -46", "last_exit_at": "2026-07-02T05:25:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.993, "last_started_at": "2026-07-02T05:23:26+09:00", "pid": 40169, "process": {"exists": false, "pid": 40169}, "restart_count": 255, "restart_count_per_hour": 21, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 256}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 253.753, "sample_count": 1235, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 359174, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:25:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T05:25:18+09:00", "last_frame_read_age_sec": 0.239, "last_frame_read_at": "2026-07-02T05:25:18+09:00", "mjpeg": {"age_sec": 0.228, "exists": true, "mtime": "2026-07-02T05:25:18.473943+09:00", "mtime_epoch": 1782937518.4739428, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165302}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:25:18+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1039.898, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 126, "pushed_frames": 126, "read_frames": 125, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.206, "pushed": 4.206, "read": 4.173}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.156, "count": 125, "max_ms": 14.522, "p95_ms": 11.609}, "gstreamer_push": {"avg_ms": 0.895, "count": 126, "max_ms": 2.366, "p95_ms": 0.938}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.776, "count": 125, "max_ms": 74.608, "p95_ms": 59.582}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.295, "count": 125, "max_ms": 89.767, "p95_ms": 86.382}, "yolo11_640_fire_11class": {"avg_ms": 52.035, "count": 125, "max_ms": 74.056, "p95_ms": 67.881}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.203, "count": 125, "max_ms": 93.577, "p95_ms": 88.25}}, "truncated_count": 0}, "processing_total": {"avg_ms": 230.032, "count": 126, "max_ms": 293.547, "p95_ms": 256.861}, "read_interval": {"avg_ms": 239.597, "count": 125, "max_ms": 302.563, "p95_ms": 265.648}, "triton_wait": {"avg_ms": 67.68, "count": 125, "max_ms": 93.587, "p95_ms": 88.31}, "visualize": {"avg_ms": 10.788, "count": 125, "max_ms": 13.042, "p95_ms": 12.077}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.954, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2993}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4391.426, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 229, "h264_decode_error_count": 411, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:25:18+09:00", "last_error_summary": "[h264 @ 0x55fd42d68740] error while decoding MB 17 44, bytestream -46", "last_exit_at": "2026-07-02T05:25:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.996, "last_started_at": "2026-07-02T05:23:26+09:00", "pid": 40169, "process": {"exists": false, "pid": 40169}, "restart_count": 255, "restart_count_per_hour": 21, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 256}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 253.753, "sample_count": 1235, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 359174, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:25:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T05:25:18+09:00", "last_frame_read_age_sec": 0.242, "last_frame_read_at": "2026-07-02T05:25:18+09:00", "mjpeg": {"age_sec": 0.231, "exists": true, "mtime": "2026-07-02T05:25:18.473943+09:00", "mtime_epoch": 1782937518.4739428, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165302}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:25:18+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1039.898, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 126, "pushed_frames": 126, "read_frames": 125, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.206, "pushed": 4.206, "read": 4.173}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.156, "count": 125, "max_ms": 14.522, "p95_ms": 11.609}, "gstreamer_push": {"avg_ms": 0.895, "count": 126, "max_ms": 2.366, "p95_ms": 0.938}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.776, "count": 125, "max_ms": 74.608, "p95_ms": 59.582}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.295, "count": 125, "max_ms": 89.767, "p95_ms": 86.382}, "yolo11_640_fire_11class": {"avg_ms": 52.035, "count": 125, "max_ms": 74.056, "p95_ms": 67.881}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.203, "count": 125, "max_ms": 93.577, "p95_ms": 88.25}}, "truncated_count": 0}, "processing_total": {"avg_ms": 230.032, "count": 126, "max_ms": 293.547, "p95_ms": 256.861}, "read_interval": {"avg_ms": 239.597, "count": 125, "max_ms": 302.563, "p95_ms": 265.648}, "triton_wait": {"avg_ms": 67.68, "count": 125, "max_ms": 93.587, "p95_ms": 88.31}, "visualize": {"avg_ms": 10.788, "count": 125, "max_ms": 13.042, "p95_ms": 12.077}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.957, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2993}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4496.715, "limit_mb": 8796093022207.996}, "detail": {"pid": 40276, "restart_count": 256, "start_count": 257}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 229, "h264_decode_error_count": 411, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:25:18+09:00", "last_error_summary": "[h264 @ 0x55fd42d68740] error while decoding MB 17 44, bytestream -46", "last_exit_at": "2026-07-02T05:25:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T05:25:21+09:00", "pid": 40276, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.047, "name": "ffmpeg", "pid": 40276, "rss_mb": 136.047, "state": "S (sleeping)", "threads": 106}, "restart_count": 256, "restart_count_per_hour": 22, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 257}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 253.753, "sample_count": 1215, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 359174, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:25:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.488, "last_frame_pushed_at": "2026-07-02T05:25:18+09:00", "last_frame_read_age_sec": 4.723, "last_frame_read_at": "2026-07-02T05:25:18+09:00", "mjpeg": {"age_sec": 4.472, "exists": true, "mtime": "2026-07-02T05:25:18.713947+09:00", "mtime_epoch": 1782937518.7139468, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 160782}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:25:23+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1039.898, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 107, "pushed_frames": 107, "read_frames": 106, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.586, "pushed": 3.586, "read": 3.552}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.219, "count": 107, "max_ms": 14.522, "p95_ms": 11.609}, "gstreamer_push": {"avg_ms": 0.899, "count": 107, "max_ms": 2.366, "p95_ms": 1.015}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.22, "count": 107, "max_ms": 74.608, "p95_ms": 59.684}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.196, "count": 107, "max_ms": 89.767, "p95_ms": 86.382}, "yolo11_640_fire_11class": {"avg_ms": 51.702, "count": 107, "max_ms": 74.056, "p95_ms": 67.881}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.918, "count": 107, "max_ms": 93.577, "p95_ms": 88.298}}, "truncated_count": 0}, "processing_total": {"avg_ms": 229.543, "count": 107, "max_ms": 289.667, "p95_ms": 256.861}, "read_interval": {"avg_ms": 239.126, "count": 106, "max_ms": 299.712, "p95_ms": 265.648}, "triton_wait": {"avg_ms": 67.569, "count": 107, "max_ms": 93.587, "p95_ms": 88.31}, "visualize": {"avg_ms": 10.784, "count": 107, "max_ms": 13.042, "p95_ms": 12.077}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.842, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2996}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4398.984, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55b804dffb00] error while decoding MB 65 43, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 230, "h264_decode_error_count": 412, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:27:10+09:00", "last_error_summary": "[h264 @ 0x55b804dffb00] error while decoding MB 65 43, bytestream -5", "last_exit_at": "2026-07-02T05:25:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.061, "last_started_at": "2026-07-02T05:25:21+09:00", "pid": 40276, "process": {"exists": false, "pid": 40276}, "restart_count": 256, "restart_count_per_hour": 22, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 257}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 256.861, "sample_count": 1218, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 359619, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:27:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-02T05:27:10+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-02T05:27:10+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-02T05:27:10.203821+09:00", "mtime_epoch": 1782937630.2038212, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165101}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:27:10+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1045.629, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 127, "pushed_frames": 127, "read_frames": 127, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.241, "pushed": 4.241, "read": 4.241}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.801, "count": 126, "max_ms": 10.442, "p95_ms": 9.45}, "gstreamer_push": {"avg_ms": 0.937, "count": 127, "max_ms": 2.941, "p95_ms": 1.766}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.548, "count": 126, "max_ms": 68.464, "p95_ms": 60.775}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.13, "count": 126, "max_ms": 99.18, "p95_ms": 89.617}, "yolo11_640_fire_11class": {"avg_ms": 54.099, "count": 126, "max_ms": 79.317, "p95_ms": 71.319}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.316, "count": 126, "max_ms": 96.528, "p95_ms": 89.599}}, "truncated_count": 0}, "processing_total": {"avg_ms": 227.518, "count": 127, "max_ms": 288.886, "p95_ms": 249.704}, "read_interval": {"avg_ms": 237.158, "count": 127, "max_ms": 298.76, "p95_ms": 260.496}, "triton_wait": {"avg_ms": 69.557, "count": 126, "max_ms": 99.191, "p95_ms": 89.627}, "visualize": {"avg_ms": 10.545, "count": 126, "max_ms": 12.017, "p95_ms": 11.223}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.949, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2996}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4397.77, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T05:27:10+09:00", "last_error_summary": "[h264 @ 0x55b804dffb00] error while decoding MB 65 43, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 230, "h264_decode_error_count": 412, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:27:10+09:00", "last_error_summary": "[h264 @ 0x55b804dffb00] error while decoding MB 65 43, bytestream -5", "last_exit_at": "2026-07-02T05:27:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.239, "last_started_at": "2026-07-02T05:25:21+09:00", "pid": 40276, "process": {"exists": false, "pid": 40276}, "restart_count": 256, "restart_count_per_hour": 22, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 257}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 256.861, "sample_count": 1218, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 359620, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:27:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T05:27:10+09:00", "last_frame_read_age_sec": 0.231, "last_frame_read_at": "2026-07-02T05:27:10+09:00", "mjpeg": {"age_sec": 0.22, "exists": true, "mtime": "2026-07-02T05:27:10.203821+09:00", "mtime_epoch": 1782937630.2038212, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165101}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:27:10+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1045.629, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 127, "pushed_frames": 127, "read_frames": 126, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.25, "pushed": 4.25, "read": 4.217}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.815, "count": 126, "max_ms": 11.065, "p95_ms": 9.462}, "gstreamer_push": {"avg_ms": 0.937, "count": 127, "max_ms": 2.941, "p95_ms": 1.766}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.386, "count": 126, "max_ms": 68.464, "p95_ms": 60.775}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.049, "count": 126, "max_ms": 99.18, "p95_ms": 89.617}, "yolo11_640_fire_11class": {"avg_ms": 54.002, "count": 126, "max_ms": 79.317, "p95_ms": 71.319}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.24, "count": 126, "max_ms": 96.528, "p95_ms": 89.599}}, "truncated_count": 0}, "processing_total": {"avg_ms": 227.532, "count": 127, "max_ms": 288.886, "p95_ms": 249.704}, "read_interval": {"avg_ms": 237.174, "count": 126, "max_ms": 298.76, "p95_ms": 260.496}, "triton_wait": {"avg_ms": 69.476, "count": 126, "max_ms": 99.191, "p95_ms": 89.627}, "visualize": {"avg_ms": 10.543, "count": 126, "max_ms": 12.017, "p95_ms": 11.223}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.879, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2996}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4397.758, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 230, "h264_decode_error_count": 412, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:27:10+09:00", "last_error_summary": "[h264 @ 0x55b804dffb00] error while decoding MB 65 43, bytestream -5", "last_exit_at": "2026-07-02T05:27:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.242, "last_started_at": "2026-07-02T05:25:21+09:00", "pid": 40276, "process": {"exists": false, "pid": 40276}, "restart_count": 256, "restart_count_per_hour": 22, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 257}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 256.861, "sample_count": 1218, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 359620, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:27:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T05:27:10+09:00", "last_frame_read_age_sec": 0.235, "last_frame_read_at": "2026-07-02T05:27:10+09:00", "mjpeg": {"age_sec": 0.224, "exists": true, "mtime": "2026-07-02T05:27:10.203821+09:00", "mtime_epoch": 1782937630.2038212, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165101}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:27:10+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1045.629, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 127, "pushed_frames": 127, "read_frames": 126, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.25, "pushed": 4.25, "read": 4.216}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.815, "count": 126, "max_ms": 11.065, "p95_ms": 9.462}, "gstreamer_push": {"avg_ms": 0.937, "count": 127, "max_ms": 2.941, "p95_ms": 1.766}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.386, "count": 126, "max_ms": 68.464, "p95_ms": 60.775}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.049, "count": 126, "max_ms": 99.18, "p95_ms": 89.617}, "yolo11_640_fire_11class": {"avg_ms": 54.002, "count": 126, "max_ms": 79.317, "p95_ms": 71.319}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.24, "count": 126, "max_ms": 96.528, "p95_ms": 89.599}}, "truncated_count": 0}, "processing_total": {"avg_ms": 227.532, "count": 127, "max_ms": 288.886, "p95_ms": 249.704}, "read_interval": {"avg_ms": 237.174, "count": 126, "max_ms": 298.76, "p95_ms": 260.496}, "triton_wait": {"avg_ms": 69.476, "count": 126, "max_ms": 99.191, "p95_ms": 89.627}, "visualize": {"avg_ms": 10.543, "count": 126, "max_ms": 12.017, "p95_ms": 11.223}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.883, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 2996}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4503.004, "limit_mb": 8796093022207.996}, "detail": {"pid": 40383, "restart_count": 257, "start_count": 258}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 230, "h264_decode_error_count": 412, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:27:10+09:00", "last_error_summary": "[h264 @ 0x55b804dffb00] error while decoding MB 65 43, bytestream -5", "last_exit_at": "2026-07-02T05:27:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T05:27:12+09:00", "pid": 40383, "process": {"exists": true, "fd_count": 5, "hwm_mb": 137.68, "name": "ffmpeg", "pid": 40383, "rss_mb": 137.68, "state": "S (sleeping)", "threads": 106}, "restart_count": 257, "restart_count_per_hour": 23, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 258}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 257.521, "sample_count": 1199, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 359620, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:27:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.353, "last_frame_pushed_at": "2026-07-02T05:27:10+09:00", "last_frame_read_age_sec": 4.581, "last_frame_read_at": "2026-07-02T05:27:10+09:00", "mjpeg": {"age_sec": 4.334, "exists": true, "mtime": "2026-07-02T05:27:10.439825+09:00", "mtime_epoch": 1782937630.4398253, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165281}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:27:14+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1045.629, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 109, "pushed_frames": 109, "read_frames": 108, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.639, "pushed": 3.639, "read": 3.606}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.737, "count": 108, "max_ms": 11.065, "p95_ms": 9.462}, "gstreamer_push": {"avg_ms": 0.942, "count": 109, "max_ms": 2.941, "p95_ms": 1.766}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.797, "count": 108, "max_ms": 68.464, "p95_ms": 60.853}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.324, "count": 108, "max_ms": 94.798, "p95_ms": 89.617}, "yolo11_640_fire_11class": {"avg_ms": 54.229, "count": 108, "max_ms": 74.369, "p95_ms": 71.319}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.658, "count": 108, "max_ms": 93.902, "p95_ms": 89.599}}, "truncated_count": 0}, "processing_total": {"avg_ms": 227.355, "count": 109, "max_ms": 288.886, "p95_ms": 249.704}, "read_interval": {"avg_ms": 237.02, "count": 108, "max_ms": 298.76, "p95_ms": 260.496}, "triton_wait": {"avg_ms": 69.779, "count": 108, "max_ms": 94.81, "p95_ms": 89.627}, "visualize": {"avg_ms": 10.496, "count": 108, "max_ms": 11.47, "p95_ms": 11.204}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.95, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3000}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4394.723, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x556e26f883c0] error while decoding MB 43 9, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 231, "h264_decode_error_count": 413, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:29:01+09:00", "last_error_summary": "[h264 @ 0x556e26f883c0] error while decoding MB 43 9, bytestream -7", "last_exit_at": "2026-07-02T05:27:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.057, "last_started_at": "2026-07-02T05:27:12+09:00", "pid": 40383, "process": {"exists": false, "pid": 40383}, "restart_count": 257, "restart_count_per_hour": 23, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 258}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 255.681, "sample_count": 1212, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 360064, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:29:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-02T05:29:01+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-02T05:29:01+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-02T05:29:01.785698+09:00", "mtime_epoch": 1782937741.7856977, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164526}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:29:01+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1039.211, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 125, "pushed_frames": 125, "read_frames": 125, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.173, "pushed": 4.173, "read": 4.173}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.086, "count": 124, "max_ms": 14.23, "p95_ms": 9.847}, "gstreamer_push": {"avg_ms": 0.958, "count": 125, "max_ms": 2.883, "p95_ms": 1.787}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.586, "count": 124, "max_ms": 70.569, "p95_ms": 60.4}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.218, "count": 124, "max_ms": 91.423, "p95_ms": 81.315}, "yolo11_640_fire_11class": {"avg_ms": 53.572, "count": 124, "max_ms": 75.786, "p95_ms": 71.175}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.888, "count": 124, "max_ms": 91.436, "p95_ms": 81.033}}, "truncated_count": 0}, "processing_total": {"avg_ms": 231.972, "count": 125, "max_ms": 307.987, "p95_ms": 255.745}, "read_interval": {"avg_ms": 241.64, "count": 125, "max_ms": 320.432, "p95_ms": 264.964}, "triton_wait": {"avg_ms": 67.889, "count": 124, "max_ms": 91.445, "p95_ms": 82.121}, "visualize": {"avg_ms": 10.85, "count": 124, "max_ms": 12.959, "p95_ms": 11.764}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.957, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3000}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4393.234, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T05:29:01+09:00", "last_error_summary": "[h264 @ 0x556e26f883c0] error while decoding MB 43 9, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 231, "h264_decode_error_count": 413, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:29:01+09:00", "last_error_summary": "[h264 @ 0x556e26f883c0] error while decoding MB 43 9, bytestream -7", "last_exit_at": "2026-07-02T05:29:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.242, "last_started_at": "2026-07-02T05:27:12+09:00", "pid": 40383, "process": {"exists": false, "pid": 40383}, "restart_count": 257, "restart_count_per_hour": 23, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 258}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 255.681, "sample_count": 1212, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 360065, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:29:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T05:29:02+09:00", "last_frame_read_age_sec": 0.237, "last_frame_read_at": "2026-07-02T05:29:01+09:00", "mjpeg": {"age_sec": 0.229, "exists": true, "mtime": "2026-07-02T05:29:01.785698+09:00", "mtime_epoch": 1782937741.7856977, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164526}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:29:02+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1039.211, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 125, "pushed_frames": 125, "read_frames": 124, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.182, "pushed": 4.182, "read": 4.149}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.087, "count": 124, "max_ms": 14.23, "p95_ms": 9.847}, "gstreamer_push": {"avg_ms": 0.95, "count": 125, "max_ms": 2.883, "p95_ms": 1.78}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.505, "count": 124, "max_ms": 70.569, "p95_ms": 60.4}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.148, "count": 124, "max_ms": 91.423, "p95_ms": 81.315}, "yolo11_640_fire_11class": {"avg_ms": 53.458, "count": 124, "max_ms": 75.786, "p95_ms": 71.175}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.872, "count": 124, "max_ms": 91.436, "p95_ms": 81.033}}, "truncated_count": 0}, "processing_total": {"avg_ms": 231.427, "count": 125, "max_ms": 307.987, "p95_ms": 251.123}, "read_interval": {"avg_ms": 241.086, "count": 124, "max_ms": 320.432, "p95_ms": 261.127}, "triton_wait": {"avg_ms": 67.819, "count": 124, "max_ms": 91.445, "p95_ms": 82.121}, "visualize": {"avg_ms": 10.847, "count": 124, "max_ms": 12.959, "p95_ms": 11.764}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.888, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3000}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4393.227, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 231, "h264_decode_error_count": 413, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:29:01+09:00", "last_error_summary": "[h264 @ 0x556e26f883c0] error while decoding MB 43 9, bytestream -7", "last_exit_at": "2026-07-02T05:29:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.246, "last_started_at": "2026-07-02T05:27:12+09:00", "pid": 40383, "process": {"exists": false, "pid": 40383}, "restart_count": 257, "restart_count_per_hour": 23, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 258}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 255.681, "sample_count": 1212, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 360065, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:29:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T05:29:02+09:00", "last_frame_read_age_sec": 0.24, "last_frame_read_at": "2026-07-02T05:29:01+09:00", "mjpeg": {"age_sec": 0.233, "exists": true, "mtime": "2026-07-02T05:29:01.785698+09:00", "mtime_epoch": 1782937741.7856977, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164526}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:29:02+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1039.211, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 125, "pushed_frames": 125, "read_frames": 124, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.182, "pushed": 4.182, "read": 4.148}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.087, "count": 124, "max_ms": 14.23, "p95_ms": 9.847}, "gstreamer_push": {"avg_ms": 0.95, "count": 125, "max_ms": 2.883, "p95_ms": 1.78}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.505, "count": 124, "max_ms": 70.569, "p95_ms": 60.4}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.148, "count": 124, "max_ms": 91.423, "p95_ms": 81.315}, "yolo11_640_fire_11class": {"avg_ms": 53.458, "count": 124, "max_ms": 75.786, "p95_ms": 71.175}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.872, "count": 124, "max_ms": 91.436, "p95_ms": 81.033}}, "truncated_count": 0}, "processing_total": {"avg_ms": 231.427, "count": 125, "max_ms": 307.987, "p95_ms": 251.123}, "read_interval": {"avg_ms": 241.086, "count": 124, "max_ms": 320.432, "p95_ms": 261.127}, "triton_wait": {"avg_ms": 67.819, "count": 124, "max_ms": 91.445, "p95_ms": 82.121}, "visualize": {"avg_ms": 10.847, "count": 124, "max_ms": 12.959, "p95_ms": 11.764}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.891, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3000}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4501.414, "limit_mb": 8796093022207.996}, "detail": {"pid": 40490, "restart_count": 258, "start_count": 259}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 231, "h264_decode_error_count": 413, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:29:01+09:00", "last_error_summary": "[h264 @ 0x556e26f883c0] error while decoding MB 43 9, bytestream -7", "last_exit_at": "2026-07-02T05:29:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T05:29:04+09:00", "pid": 40490, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.238, "name": "ffmpeg", "pid": 40490, "rss_mb": 139.238, "state": "S (sleeping)", "threads": 106}, "restart_count": 258, "restart_count_per_hour": 24, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 259}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 255.496, "sample_count": 1192, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 360065, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:29:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.548, "last_frame_pushed_at": "2026-07-02T05:29:02+09:00", "last_frame_read_age_sec": 4.781, "last_frame_read_at": "2026-07-02T05:29:01+09:00", "mjpeg": {"age_sec": 4.53, "exists": true, "mtime": "2026-07-02T05:29:02.029702+09:00", "mtime_epoch": 1782937742.0297017, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 157645}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:29:06+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1039.211, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 106, "pushed_frames": 106, "read_frames": 105, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.551, "pushed": 3.551, "read": 3.517}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.034, "count": 106, "max_ms": 14.23, "p95_ms": 9.815}, "gstreamer_push": {"avg_ms": 0.963, "count": 106, "max_ms": 2.883, "p95_ms": 1.787}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.672, "count": 106, "max_ms": 67.565, "p95_ms": 60.4}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.19, "count": 106, "max_ms": 91.423, "p95_ms": 79.581}, "yolo11_640_fire_11class": {"avg_ms": 53.567, "count": 106, "max_ms": 75.786, "p95_ms": 71.175}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.171, "count": 106, "max_ms": 91.436, "p95_ms": 81.033}}, "truncated_count": 0}, "processing_total": {"avg_ms": 231.313, "count": 106, "max_ms": 307.987, "p95_ms": 251.123}, "read_interval": {"avg_ms": 240.985, "count": 105, "max_ms": 320.432, "p95_ms": 261.127}, "triton_wait": {"avg_ms": 67.932, "count": 106, "max_ms": 91.445, "p95_ms": 82.121}, "visualize": {"avg_ms": 10.817, "count": 106, "max_ms": 12.809, "p95_ms": 11.718}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.854, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3004}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4398.602, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55b95c458600] error while decoding MB 65 24, bytestream -8", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 232, "h264_decode_error_count": 414, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:30:55+09:00", "last_error_summary": "[h264 @ 0x55b95c458600] error while decoding MB 65 24, bytestream -8", "last_exit_at": "2026-07-02T05:29:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.293, "last_started_at": "2026-07-02T05:29:04+09:00", "pid": 40490, "process": {"exists": false, "pid": 40490}, "restart_count": 258, "restart_count_per_hour": 24, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 259}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 270.918, "sample_count": 1197, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 360506, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:30:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-02T05:30:55+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-02T05:30:55+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-02T05:30:55.811615+09:00", "mtime_epoch": 1782937855.8116155, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169743}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:30:55+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1043.359, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 117, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.911, "pushed": 3.911, "read": 3.911}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.866, "count": 116, "max_ms": 12.409, "p95_ms": 10.641}, "gstreamer_push": {"avg_ms": 0.89, "count": 117, "max_ms": 1.832, "p95_ms": 0.93}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.974, "count": 116, "max_ms": 71.142, "p95_ms": 61.719}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.578, "count": 116, "max_ms": 95.379, "p95_ms": 92.507}, "yolo11_640_fire_11class": {"avg_ms": 53.979, "count": 116, "max_ms": 76.537, "p95_ms": 72.597}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.623, "count": 116, "max_ms": 96.156, "p95_ms": 93.079}}, "truncated_count": 0}, "processing_total": {"avg_ms": 247.603, "count": 117, "max_ms": 359.845, "p95_ms": 279.634}, "read_interval": {"avg_ms": 257.181, "count": 117, "max_ms": 371.347, "p95_ms": 288.796}, "triton_wait": {"avg_ms": 72.814, "count": 116, "max_ms": 96.167, "p95_ms": 93.278}, "visualize": {"avg_ms": 11.283, "count": 116, "max_ms": 12.969, "p95_ms": 12.129}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.914, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3004}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4397.398, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T05:30:55+09:00", "last_error_summary": "[h264 @ 0x55b95c458600] error while decoding MB 65 24, bytestream -8", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 232, "h264_decode_error_count": 414, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:30:55+09:00", "last_error_summary": "[h264 @ 0x55b95c458600] error while decoding MB 65 24, bytestream -8", "last_exit_at": "2026-07-02T05:30:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.516, "last_started_at": "2026-07-02T05:29:04+09:00", "pid": 40490, "process": {"exists": false, "pid": 40490}, "restart_count": 258, "restart_count_per_hour": 24, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 259}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 270.918, "sample_count": 1197, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 360507, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:30:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T05:30:56+09:00", "last_frame_read_age_sec": 0.273, "last_frame_read_at": "2026-07-02T05:30:55+09:00", "mjpeg": {"age_sec": 0.263, "exists": true, "mtime": "2026-07-02T05:30:55.811615+09:00", "mtime_epoch": 1782937855.8116155, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169743}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:30:56+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1043.359, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.914, "pushed": 3.914, "read": 3.88}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.874, "count": 116, "max_ms": 12.409, "p95_ms": 10.641}, "gstreamer_push": {"avg_ms": 0.89, "count": 117, "max_ms": 1.832, "p95_ms": 0.93}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.021, "count": 116, "max_ms": 71.142, "p95_ms": 61.719}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.761, "count": 116, "max_ms": 95.379, "p95_ms": 92.507}, "yolo11_640_fire_11class": {"avg_ms": 54.168, "count": 116, "max_ms": 76.537, "p95_ms": 72.6}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.79, "count": 116, "max_ms": 96.156, "p95_ms": 93.079}}, "truncated_count": 0}, "processing_total": {"avg_ms": 247.95, "count": 117, "max_ms": 359.845, "p95_ms": 279.634}, "read_interval": {"avg_ms": 257.348, "count": 116, "max_ms": 371.347, "p95_ms": 288.796}, "triton_wait": {"avg_ms": 72.997, "count": 116, "max_ms": 96.167, "p95_ms": 93.278}, "visualize": {"avg_ms": 11.287, "count": 116, "max_ms": 12.969, "p95_ms": 12.129}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.895, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3004}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4397.391, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 232, "h264_decode_error_count": 414, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:30:55+09:00", "last_error_summary": "[h264 @ 0x55b95c458600] error while decoding MB 65 24, bytestream -8", "last_exit_at": "2026-07-02T05:30:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 111.519, "last_started_at": "2026-07-02T05:29:04+09:00", "pid": 40490, "process": {"exists": false, "pid": 40490}, "restart_count": 258, "restart_count_per_hour": 24, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 259}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 270.918, "sample_count": 1197, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 360507, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:30:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T05:30:56+09:00", "last_frame_read_age_sec": 0.276, "last_frame_read_at": "2026-07-02T05:30:55+09:00", "mjpeg": {"age_sec": 0.266, "exists": true, "mtime": "2026-07-02T05:30:55.811615+09:00", "mtime_epoch": 1782937855.8116155, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169743}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:30:56+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1043.359, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 116, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.913, "pushed": 3.913, "read": 3.88}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.874, "count": 116, "max_ms": 12.409, "p95_ms": 10.641}, "gstreamer_push": {"avg_ms": 0.89, "count": 117, "max_ms": 1.832, "p95_ms": 0.93}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.021, "count": 116, "max_ms": 71.142, "p95_ms": 61.719}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.761, "count": 116, "max_ms": 95.379, "p95_ms": 92.507}, "yolo11_640_fire_11class": {"avg_ms": 54.168, "count": 116, "max_ms": 76.537, "p95_ms": 72.6}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.79, "count": 116, "max_ms": 96.156, "p95_ms": 93.079}}, "truncated_count": 0}, "processing_total": {"avg_ms": 247.95, "count": 117, "max_ms": 359.845, "p95_ms": 279.634}, "read_interval": {"avg_ms": 257.348, "count": 116, "max_ms": 371.347, "p95_ms": 288.796}, "triton_wait": {"avg_ms": 72.997, "count": 116, "max_ms": 96.167, "p95_ms": 93.278}, "visualize": {"avg_ms": 11.287, "count": 116, "max_ms": 12.969, "p95_ms": 12.129}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.898, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3004}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4497.262, "limit_mb": 8796093022207.996}, "detail": {"pid": 40597, "restart_count": 259, "start_count": 260}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 232, "h264_decode_error_count": 414, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:30:55+09:00", "last_error_summary": "[h264 @ 0x55b95c458600] error while decoding MB 65 24, bytestream -8", "last_exit_at": "2026-07-02T05:30:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T05:30:58+09:00", "pid": 40597, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.191, "name": "ffmpeg", "pid": 40597, "rss_mb": 139.191, "state": "S (sleeping)", "threads": 106}, "restart_count": 259, "restart_count_per_hour": 25, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 260}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 271.039, "sample_count": 1179, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 360507, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:30:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.431, "last_frame_pushed_at": "2026-07-02T05:30:56+09:00", "last_frame_read_age_sec": 4.699, "last_frame_read_at": "2026-07-02T05:30:55+09:00", "mjpeg": {"age_sec": 4.413, "exists": true, "mtime": "2026-07-02T05:30:56.087620+09:00", "mtime_epoch": 1782937856.0876203, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169856}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:31:00+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1043.359, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 98, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.323, "pushed": 3.323, "read": 3.29}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.986, "count": 98, "max_ms": 12.409, "p95_ms": 10.71}, "gstreamer_push": {"avg_ms": 0.885, "count": 99, "max_ms": 1.8, "p95_ms": 0.93}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.334, "count": 98, "max_ms": 71.142, "p95_ms": 61.795}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.756, "count": 98, "max_ms": 95.379, "p95_ms": 92.507}, "yolo11_640_fire_11class": {"avg_ms": 54.372, "count": 98, "max_ms": 76.537, "p95_ms": 73.472}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.873, "count": 98, "max_ms": 96.156, "p95_ms": 93.059}}, "truncated_count": 0}, "processing_total": {"avg_ms": 250.101, "count": 99, "max_ms": 359.845, "p95_ms": 280.715}, "read_interval": {"avg_ms": 259.506, "count": 98, "max_ms": 371.347, "p95_ms": 289.787}, "triton_wait": {"avg_ms": 72.943, "count": 98, "max_ms": 96.167, "p95_ms": 93.09}, "visualize": {"avg_ms": 11.3, "count": 98, "max_ms": 12.969, "p95_ms": 12.351}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.789, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3007}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4399.184, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55d78f935000] error while decoding MB 79 20, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 233, "h264_decode_error_count": 415, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:32:46+09:00", "last_error_summary": "[h264 @ 0x55d78f935000] error while decoding MB 79 20, bytestream -5", "last_exit_at": "2026-07-02T05:30:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.87, "last_started_at": "2026-07-02T05:30:58+09:00", "pid": 40597, "process": {"exists": false, "pid": 40597}, "restart_count": 259, "restart_count_per_hour": 24, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 260}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 270.273, "sample_count": 1207, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 360959, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:32:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-02T05:32:46+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-02T05:32:46+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-02T05:32:46.325475+09:00", "mtime_epoch": 1782937966.3254747, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165758}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:32:46+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1043.492, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 129, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.303, "pushed": 4.303, "read": 4.303}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.511, "count": 128, "max_ms": 12.804, "p95_ms": 10.014}, "gstreamer_push": {"avg_ms": 0.925, "count": 129, "max_ms": 2.9, "p95_ms": 1.757}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.523, "count": 128, "max_ms": 69.544, "p95_ms": 61.321}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.329, "count": 128, "max_ms": 94.21, "p95_ms": 88.791}, "yolo11_640_fire_11class": {"avg_ms": 54.408, "count": 128, "max_ms": 73.902, "p95_ms": 71.942}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.747, "count": 128, "max_ms": 94.986, "p95_ms": 89.9}}, "truncated_count": 0}, "processing_total": {"avg_ms": 224.102, "count": 129, "max_ms": 300.417, "p95_ms": 254.006}, "read_interval": {"avg_ms": 233.685, "count": 129, "max_ms": 312.211, "p95_ms": 264.307}, "triton_wait": {"avg_ms": 69.856, "count": 128, "max_ms": 94.995, "p95_ms": 89.928}, "visualize": {"avg_ms": 10.576, "count": 128, "max_ms": 16.142, "p95_ms": 14.427}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.978, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3008}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4398.332, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T05:32:46+09:00", "last_error_summary": "[h264 @ 0x55d78f935000] error while decoding MB 79 20, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 233, "h264_decode_error_count": 415, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:32:46+09:00", "last_error_summary": "[h264 @ 0x55d78f935000] error while decoding MB 79 20, bytestream -5", "last_exit_at": "2026-07-02T05:32:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.068, "last_started_at": "2026-07-02T05:30:58+09:00", "pid": 40597, "process": {"exists": false, "pid": 40597}, "restart_count": 259, "restart_count_per_hour": 24, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 260}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 270.273, "sample_count": 1208, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 360960, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:32:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T05:32:46+09:00", "last_frame_read_age_sec": 0.25, "last_frame_read_at": "2026-07-02T05:32:46+09:00", "mjpeg": {"age_sec": 0.242, "exists": true, "mtime": "2026-07-02T05:32:46.325475+09:00", "mtime_epoch": 1782937966.3254747, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165758}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:32:46+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1043.492, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 128, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.311, "pushed": 4.311, "read": 4.278}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.515, "count": 128, "max_ms": 12.804, "p95_ms": 10.014}, "gstreamer_push": {"avg_ms": 0.925, "count": 129, "max_ms": 2.9, "p95_ms": 1.757}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.378, "count": 128, "max_ms": 69.544, "p95_ms": 61.321}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.309, "count": 128, "max_ms": 94.21, "p95_ms": 88.791}, "yolo11_640_fire_11class": {"avg_ms": 54.336, "count": 128, "max_ms": 73.902, "p95_ms": 71.942}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.711, "count": 128, "max_ms": 94.986, "p95_ms": 89.9}}, "truncated_count": 0}, "processing_total": {"avg_ms": 224.303, "count": 129, "max_ms": 300.417, "p95_ms": 254.006}, "read_interval": {"avg_ms": 233.723, "count": 128, "max_ms": 312.211, "p95_ms": 264.307}, "triton_wait": {"avg_ms": 69.835, "count": 128, "max_ms": 94.995, "p95_ms": 89.928}, "visualize": {"avg_ms": 10.572, "count": 128, "max_ms": 16.142, "p95_ms": 14.427}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.922, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3008}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4398.254, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 233, "h264_decode_error_count": 415, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:32:46+09:00", "last_error_summary": "[h264 @ 0x55d78f935000] error while decoding MB 79 20, bytestream -5", "last_exit_at": "2026-07-02T05:32:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.071, "last_started_at": "2026-07-02T05:30:58+09:00", "pid": 40597, "process": {"exists": false, "pid": 40597}, "restart_count": 259, "restart_count_per_hour": 24, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 260}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 270.273, "sample_count": 1208, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 360960, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:32:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T05:32:46+09:00", "last_frame_read_age_sec": 0.253, "last_frame_read_at": "2026-07-02T05:32:46+09:00", "mjpeg": {"age_sec": 0.246, "exists": true, "mtime": "2026-07-02T05:32:46.325475+09:00", "mtime_epoch": 1782937966.3254747, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 165758}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:32:46+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1043.492, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 129, "pushed_frames": 129, "read_frames": 128, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.311, "pushed": 4.311, "read": 4.277}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.515, "count": 128, "max_ms": 12.804, "p95_ms": 10.014}, "gstreamer_push": {"avg_ms": 0.925, "count": 129, "max_ms": 2.9, "p95_ms": 1.757}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.378, "count": 128, "max_ms": 69.544, "p95_ms": 61.321}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.309, "count": 128, "max_ms": 94.21, "p95_ms": 88.791}, "yolo11_640_fire_11class": {"avg_ms": 54.336, "count": 128, "max_ms": 73.902, "p95_ms": 71.942}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.711, "count": 128, "max_ms": 94.986, "p95_ms": 89.9}}, "truncated_count": 0}, "processing_total": {"avg_ms": 224.303, "count": 129, "max_ms": 300.417, "p95_ms": 254.006}, "read_interval": {"avg_ms": 233.723, "count": 128, "max_ms": 312.211, "p95_ms": 264.307}, "triton_wait": {"avg_ms": 69.835, "count": 128, "max_ms": 94.995, "p95_ms": 89.928}, "visualize": {"avg_ms": 10.572, "count": 128, "max_ms": 16.142, "p95_ms": 14.427}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.926, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3008}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4506.316, "limit_mb": 8796093022207.996}, "detail": {"pid": 40704, "restart_count": 260, "start_count": 261}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 233, "h264_decode_error_count": 415, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:32:46+09:00", "last_error_summary": "[h264 @ 0x55d78f935000] error while decoding MB 79 20, bytestream -5", "last_exit_at": "2026-07-02T05:32:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T05:32:48+09:00", "pid": 40704, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.344, "name": "ffmpeg", "pid": 40704, "rss_mb": 139.344, "state": "S (sleeping)", "threads": 106}, "restart_count": 260, "restart_count_per_hour": 25, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 261}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 270.273, "sample_count": 1190, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 360960, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:32:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.334, "last_frame_pushed_at": "2026-07-02T05:32:46+09:00", "last_frame_read_age_sec": 4.581, "last_frame_read_at": "2026-07-02T05:32:46+09:00", "mjpeg": {"age_sec": 4.318, "exists": true, "mtime": "2026-07-02T05:32:46.581479+09:00", "mtime_epoch": 1782937966.5814788, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 162471}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:32:50+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1043.492, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 110, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.719, "pushed": 3.719, "read": 3.686}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.521, "count": 111, "max_ms": 12.804, "p95_ms": 10.02}, "gstreamer_push": {"avg_ms": 0.915, "count": 111, "max_ms": 1.887, "p95_ms": 1.757}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.816, "count": 111, "max_ms": 69.544, "p95_ms": 61.684}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.354, "count": 111, "max_ms": 94.21, "p95_ms": 89.218}, "yolo11_640_fire_11class": {"avg_ms": 54.609, "count": 111, "max_ms": 73.902, "p95_ms": 72.152}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.922, "count": 111, "max_ms": 94.986, "p95_ms": 89.915}}, "truncated_count": 0}, "processing_total": {"avg_ms": 222.312, "count": 111, "max_ms": 300.417, "p95_ms": 254.006}, "read_interval": {"avg_ms": 231.616, "count": 110, "max_ms": 312.211, "p95_ms": 264.307}, "triton_wait": {"avg_ms": 69.887, "count": 111, "max_ms": 94.995, "p95_ms": 90.819}, "visualize": {"avg_ms": 10.3, "count": 111, "max_ms": 15.716, "p95_ms": 12.945}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.846, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3011}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4402.656, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x560e03fd0b80] error while decoding MB 12 28, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 234, "h264_decode_error_count": 416, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:34:37+09:00", "last_error_summary": "[h264 @ 0x560e03fd0b80] error while decoding MB 12 28, bytestream -9", "last_exit_at": "2026-07-02T05:32:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.392, "last_started_at": "2026-07-02T05:32:48+09:00", "pid": 40704, "process": {"exists": false, "pid": 40704}, "restart_count": 260, "restart_count_per_hour": 25, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 261}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 266.519, "sample_count": 1214, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 361408, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:34:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-02T05:34:37+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-02T05:34:37+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-02T05:34:37.247341+09:00", "mtime_epoch": 1782938077.2473407, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173238}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:34:37+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1046.469, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 124, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.154, "pushed": 4.154, "read": 4.154}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.148, "count": 124, "max_ms": 15.561, "p95_ms": 11.61}, "gstreamer_push": {"avg_ms": 0.89, "count": 124, "max_ms": 1.838, "p95_ms": 1.0}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.076, "count": 124, "max_ms": 74.461, "p95_ms": 62.98}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.639, "count": 124, "max_ms": 93.447, "p95_ms": 88.665}, "yolo11_640_fire_11class": {"avg_ms": 54.586, "count": 124, "max_ms": 76.143, "p95_ms": 72.614}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.11, "count": 124, "max_ms": 94.22, "p95_ms": 90.976}}, "truncated_count": 0}, "processing_total": {"avg_ms": 232.289, "count": 124, "max_ms": 314.426, "p95_ms": 259.03}, "read_interval": {"avg_ms": 241.979, "count": 124, "max_ms": 324.426, "p95_ms": 270.217}, "triton_wait": {"avg_ms": 71.026, "count": 124, "max_ms": 94.23, "p95_ms": 91.596}, "visualize": {"avg_ms": 10.82, "count": 124, "max_ms": 15.514, "p95_ms": 12.96}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.851, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3011}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4401.766, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T05:34:37+09:00", "last_error_summary": "[h264 @ 0x560e03fd0b80] error while decoding MB 12 28, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 234, "h264_decode_error_count": 416, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:34:37+09:00", "last_error_summary": "[h264 @ 0x560e03fd0b80] error while decoding MB 12 28, bytestream -9", "last_exit_at": "2026-07-02T05:34:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.555, "last_started_at": "2026-07-02T05:32:48+09:00", "pid": 40704, "process": {"exists": false, "pid": 40704}, "restart_count": 260, "restart_count_per_hour": 25, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 261}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 266.519, "sample_count": 1214, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 361409, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:34:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T05:34:37+09:00", "last_frame_read_age_sec": 0.214, "last_frame_read_at": "2026-07-02T05:34:37+09:00", "mjpeg": {"age_sec": 0.206, "exists": true, "mtime": "2026-07-02T05:34:37.247341+09:00", "mtime_epoch": 1782938077.2473407, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173238}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:34:37+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1046.469, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.165, "pushed": 4.165, "read": 4.132}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.151, "count": 124, "max_ms": 15.561, "p95_ms": 11.61}, "gstreamer_push": {"avg_ms": 0.891, "count": 124, "max_ms": 1.838, "p95_ms": 1.0}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.118, "count": 124, "max_ms": 74.461, "p95_ms": 62.98}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.681, "count": 124, "max_ms": 93.447, "p95_ms": 88.665}, "yolo11_640_fire_11class": {"avg_ms": 54.659, "count": 124, "max_ms": 76.143, "p95_ms": 72.614}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.177, "count": 124, "max_ms": 94.22, "p95_ms": 90.976}}, "truncated_count": 0}, "processing_total": {"avg_ms": 232.318, "count": 124, "max_ms": 314.426, "p95_ms": 259.03}, "read_interval": {"avg_ms": 242.182, "count": 123, "max_ms": 324.426, "p95_ms": 270.217}, "triton_wait": {"avg_ms": 71.068, "count": 124, "max_ms": 94.23, "p95_ms": 91.596}, "visualize": {"avg_ms": 10.818, "count": 124, "max_ms": 15.514, "p95_ms": 12.96}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.769, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3011}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4401.883, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 234, "h264_decode_error_count": 416, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:34:37+09:00", "last_error_summary": "[h264 @ 0x560e03fd0b80] error while decoding MB 12 28, bytestream -9", "last_exit_at": "2026-07-02T05:34:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.558, "last_started_at": "2026-07-02T05:32:48+09:00", "pid": 40704, "process": {"exists": false, "pid": 40704}, "restart_count": 260, "restart_count_per_hour": 25, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 261}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 266.519, "sample_count": 1214, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 361409, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:34:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T05:34:37+09:00", "last_frame_read_age_sec": 0.217, "last_frame_read_at": "2026-07-02T05:34:37+09:00", "mjpeg": {"age_sec": 0.209, "exists": true, "mtime": "2026-07-02T05:34:37.247341+09:00", "mtime_epoch": 1782938077.2473407, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173238}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:34:37+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1046.469, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.165, "pushed": 4.165, "read": 4.131}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.151, "count": 124, "max_ms": 15.561, "p95_ms": 11.61}, "gstreamer_push": {"avg_ms": 0.891, "count": 124, "max_ms": 1.838, "p95_ms": 1.0}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.118, "count": 124, "max_ms": 74.461, "p95_ms": 62.98}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.681, "count": 124, "max_ms": 93.447, "p95_ms": 88.665}, "yolo11_640_fire_11class": {"avg_ms": 54.659, "count": 124, "max_ms": 76.143, "p95_ms": 72.614}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.177, "count": 124, "max_ms": 94.22, "p95_ms": 90.976}}, "truncated_count": 0}, "processing_total": {"avg_ms": 232.318, "count": 124, "max_ms": 314.426, "p95_ms": 259.03}, "read_interval": {"avg_ms": 242.182, "count": 123, "max_ms": 324.426, "p95_ms": 270.217}, "triton_wait": {"avg_ms": 71.068, "count": 124, "max_ms": 94.23, "p95_ms": 91.596}, "visualize": {"avg_ms": 10.818, "count": 124, "max_ms": 15.514, "p95_ms": 12.96}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.772, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3011}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4513.016, "limit_mb": 8796093022207.996}, "detail": {"pid": 40811, "restart_count": 261, "start_count": 262}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 234, "h264_decode_error_count": 416, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:34:37+09:00", "last_error_summary": "[h264 @ 0x560e03fd0b80] error while decoding MB 12 28, bytestream -9", "last_exit_at": "2026-07-02T05:34:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T05:34:39+09:00", "pid": 40811, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.973, "name": "ffmpeg", "pid": 40811, "rss_mb": 138.973, "state": "S (sleeping)", "threads": 106}, "restart_count": 261, "restart_count_per_hour": 26, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 262}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 266.519, "sample_count": 1198, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 361409, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:34:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.105, "last_frame_pushed_at": "2026-07-02T05:34:37+09:00", "last_frame_read_age_sec": 4.316, "last_frame_read_at": "2026-07-02T05:34:37+09:00", "mjpeg": {"age_sec": 4.088, "exists": true, "mtime": "2026-07-02T05:34:37.467345+09:00", "mtime_epoch": 1782938077.4673445, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 167961}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:34:41+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1046.469, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 107, "pushed_frames": 107, "read_frames": 106, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.594, "pushed": 3.594, "read": 3.56}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.138, "count": 107, "max_ms": 15.561, "p95_ms": 11.61}, "gstreamer_push": {"avg_ms": 0.894, "count": 107, "max_ms": 1.838, "p95_ms": 1.0}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.009, "count": 107, "max_ms": 74.461, "p95_ms": 62.98}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.861, "count": 107, "max_ms": 93.447, "p95_ms": 88.841}, "yolo11_640_fire_11class": {"avg_ms": 54.869, "count": 107, "max_ms": 76.143, "p95_ms": 72.614}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.335, "count": 107, "max_ms": 93.973, "p95_ms": 90.976}}, "truncated_count": 0}, "processing_total": {"avg_ms": 232.691, "count": 107, "max_ms": 314.426, "p95_ms": 260.21}, "read_interval": {"avg_ms": 242.567, "count": 106, "max_ms": 324.426, "p95_ms": 270.217}, "triton_wait": {"avg_ms": 71.184, "count": 107, "max_ms": 93.982, "p95_ms": 91.596}, "visualize": {"avg_ms": 10.88, "count": 107, "max_ms": 15.514, "p95_ms": 13.047}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.773, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3015}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4395.531, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5621c6fa2780] error while decoding MB 26 13, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 235, "h264_decode_error_count": 417, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:36:28+09:00", "last_error_summary": "[h264 @ 0x5621c6fa2780] error while decoding MB 26 13, bytestream -5", "last_exit_at": "2026-07-02T05:34:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.292, "last_started_at": "2026-07-02T05:34:39+09:00", "pid": 40811, "process": {"exists": false, "pid": 40811}, "restart_count": 261, "restart_count_per_hour": 26, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 262}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 253.533, "sample_count": 1232, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 361859, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:36:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-02T05:36:28+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-02T05:36:28+09:00", "mjpeg": {"age_sec": 0.057, "exists": true, "mtime": "2026-07-02T05:36:28.789217+09:00", "mtime_epoch": 1782938188.7892175, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172790}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:36:28+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1042.184, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 124, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.138, "pushed": 4.138, "read": 4.138}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.067, "count": 123, "max_ms": 12.396, "p95_ms": 9.646}, "gstreamer_push": {"avg_ms": 0.959, "count": 124, "max_ms": 2.974, "p95_ms": 1.793}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.222, "count": 123, "max_ms": 68.312, "p95_ms": 58.354}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.513, "count": 123, "max_ms": 91.187, "p95_ms": 81.685}, "yolo11_640_fire_11class": {"avg_ms": 53.105, "count": 123, "max_ms": 76.831, "p95_ms": 68.791}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.246, "count": 123, "max_ms": 89.957, "p95_ms": 79.01}}, "truncated_count": 0}, "processing_total": {"avg_ms": 233.142, "count": 124, "max_ms": 310.071, "p95_ms": 255.085}, "read_interval": {"avg_ms": 243.152, "count": 124, "max_ms": 322.75, "p95_ms": 264.658}, "triton_wait": {"avg_ms": 68.626, "count": 123, "max_ms": 91.198, "p95_ms": 83.212}, "visualize": {"avg_ms": 11.025, "count": 123, "max_ms": 17.448, "p95_ms": 13.169}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.968, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3015}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4394.926, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T05:36:28+09:00", "last_error_summary": "[h264 @ 0x5621c6fa2780] error while decoding MB 26 13, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 235, "h264_decode_error_count": 417, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:36:28+09:00", "last_error_summary": "[h264 @ 0x5621c6fa2780] error while decoding MB 26 13, bytestream -5", "last_exit_at": "2026-07-02T05:36:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.479, "last_started_at": "2026-07-02T05:34:39+09:00", "pid": 40811, "process": {"exists": false, "pid": 40811}, "restart_count": 261, "restart_count_per_hour": 26, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 262}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 253.533, "sample_count": 1232, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 361860, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:36:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T05:36:29+09:00", "last_frame_read_age_sec": 0.239, "last_frame_read_at": "2026-07-02T05:36:28+09:00", "mjpeg": {"age_sec": 0.243, "exists": true, "mtime": "2026-07-02T05:36:28.789217+09:00", "mtime_epoch": 1782938188.7892175, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172790}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:36:29+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1042.184, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.146, "pushed": 4.146, "read": 4.113}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.065, "count": 123, "max_ms": 12.396, "p95_ms": 9.646}, "gstreamer_push": {"avg_ms": 0.942, "count": 124, "max_ms": 2.974, "p95_ms": 1.777}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.229, "count": 123, "max_ms": 68.312, "p95_ms": 58.354}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.501, "count": 123, "max_ms": 91.187, "p95_ms": 81.685}, "yolo11_640_fire_11class": {"avg_ms": 53.127, "count": 123, "max_ms": 76.831, "p95_ms": 68.791}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.26, "count": 123, "max_ms": 89.957, "p95_ms": 79.01}}, "truncated_count": 0}, "processing_total": {"avg_ms": 233.122, "count": 124, "max_ms": 310.071, "p95_ms": 255.085}, "read_interval": {"avg_ms": 243.109, "count": 123, "max_ms": 322.75, "p95_ms": 264.658}, "triton_wait": {"avg_ms": 68.613, "count": 123, "max_ms": 91.198, "p95_ms": 83.212}, "visualize": {"avg_ms": 11.027, "count": 123, "max_ms": 17.448, "p95_ms": 13.169}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.909, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3015}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4394.973, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 235, "h264_decode_error_count": 417, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:36:28+09:00", "last_error_summary": "[h264 @ 0x5621c6fa2780] error while decoding MB 26 13, bytestream -5", "last_exit_at": "2026-07-02T05:36:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.483, "last_started_at": "2026-07-02T05:34:39+09:00", "pid": 40811, "process": {"exists": false, "pid": 40811}, "restart_count": 261, "restart_count_per_hour": 26, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 262}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 253.533, "sample_count": 1232, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 361860, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:36:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-02T05:36:29+09:00", "last_frame_read_age_sec": 0.244, "last_frame_read_at": "2026-07-02T05:36:28+09:00", "mjpeg": {"age_sec": 0.248, "exists": true, "mtime": "2026-07-02T05:36:28.789217+09:00", "mtime_epoch": 1782938188.7892175, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172790}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:36:29+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1042.184, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.145, "pushed": 4.145, "read": 4.112}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.065, "count": 123, "max_ms": 12.396, "p95_ms": 9.646}, "gstreamer_push": {"avg_ms": 0.942, "count": 124, "max_ms": 2.974, "p95_ms": 1.777}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.229, "count": 123, "max_ms": 68.312, "p95_ms": 58.354}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.501, "count": 123, "max_ms": 91.187, "p95_ms": 81.685}, "yolo11_640_fire_11class": {"avg_ms": 53.127, "count": 123, "max_ms": 76.831, "p95_ms": 68.791}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.26, "count": 123, "max_ms": 89.957, "p95_ms": 79.01}}, "truncated_count": 0}, "processing_total": {"avg_ms": 233.122, "count": 124, "max_ms": 310.071, "p95_ms": 255.085}, "read_interval": {"avg_ms": 243.109, "count": 123, "max_ms": 322.75, "p95_ms": 264.658}, "triton_wait": {"avg_ms": 68.613, "count": 123, "max_ms": 91.198, "p95_ms": 83.212}, "visualize": {"avg_ms": 11.027, "count": 123, "max_ms": 17.448, "p95_ms": 13.169}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.913, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3015}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4510.938, "limit_mb": 8796093022207.996}, "detail": {"pid": 40918, "restart_count": 262, "start_count": 263}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 235, "h264_decode_error_count": 417, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:36:28+09:00", "last_error_summary": "[h264 @ 0x5621c6fa2780] error while decoding MB 26 13, bytestream -5", "last_exit_at": "2026-07-02T05:36:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T05:36:31+09:00", "pid": 40918, "process": {"exists": true, "fd_count": 5, "hwm_mb": 142.043, "name": "ffmpeg", "pid": 40918, "rss_mb": 142.043, "state": "S (sleeping)", "threads": 106}, "restart_count": 262, "restart_count_per_hour": 27, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 263}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 253.667, "sample_count": 1213, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 361860, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:36:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.54, "last_frame_pushed_at": "2026-07-02T05:36:29+09:00", "last_frame_read_age_sec": 4.775, "last_frame_read_at": "2026-07-02T05:36:28+09:00", "mjpeg": {"age_sec": 4.531, "exists": true, "mtime": "2026-07-02T05:36:29.037222+09:00", "mtime_epoch": 1782938189.0372217, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173173}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:36:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1042.184, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 105, "pushed_frames": 105, "read_frames": 104, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.518, "pushed": 3.518, "read": 3.485}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.05, "count": 105, "max_ms": 12.396, "p95_ms": 9.798}, "gstreamer_push": {"avg_ms": 0.938, "count": 105, "max_ms": 2.974, "p95_ms": 1.777}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.14, "count": 105, "max_ms": 68.312, "p95_ms": 58.479}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.431, "count": 105, "max_ms": 91.187, "p95_ms": 83.203}, "yolo11_640_fire_11class": {"avg_ms": 53.027, "count": 105, "max_ms": 76.831, "p95_ms": 69.018}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.372, "count": 105, "max_ms": 89.957, "p95_ms": 80.909}}, "truncated_count": 0}, "processing_total": {"avg_ms": 233.153, "count": 105, "max_ms": 310.071, "p95_ms": 255.085}, "read_interval": {"avg_ms": 243.201, "count": 104, "max_ms": 322.75, "p95_ms": 264.658}, "triton_wait": {"avg_ms": 68.561, "count": 105, "max_ms": 91.198, "p95_ms": 83.38}, "visualize": {"avg_ms": 11.075, "count": 105, "max_ms": 17.448, "p95_ms": 13.438}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.845, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3019}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4399.004, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55ad7e1af100] error while decoding MB 22 38, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 236, "h264_decode_error_count": 418, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:38:21+09:00", "last_error_summary": "[h264 @ 0x55ad7e1af100] error while decoding MB 22 38, bytestream -5", "last_exit_at": "2026-07-02T05:36:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.455, "last_started_at": "2026-07-02T05:36:31+09:00", "pid": 40918, "process": {"exists": false, "pid": 40918}, "restart_count": 262, "restart_count_per_hour": 27, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 263}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 255.109, "sample_count": 1221, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 362307, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:38:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-02T05:38:20+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-02T05:38:20+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-02T05:38:20.983105+09:00", "mtime_epoch": 1782938300.9831054, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175236}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:38:21+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1043.859, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 124, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.164, "pushed": 4.164, "read": 4.164}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.087, "count": 124, "max_ms": 14.653, "p95_ms": 10.501}, "gstreamer_push": {"avg_ms": 0.946, "count": 124, "max_ms": 2.859, "p95_ms": 1.767}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.003, "count": 124, "max_ms": 71.245, "p95_ms": 58.516}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.215, "count": 124, "max_ms": 92.862, "p95_ms": 85.602}, "yolo11_640_fire_11class": {"avg_ms": 52.264, "count": 124, "max_ms": 72.979, "p95_ms": 69.58}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.602, "count": 124, "max_ms": 92.851, "p95_ms": 87.604}}, "truncated_count": 0}, "processing_total": {"avg_ms": 231.968, "count": 124, "max_ms": 317.974, "p95_ms": 279.21}, "read_interval": {"avg_ms": 241.62, "count": 124, "max_ms": 328.429, "p95_ms": 290.666}, "triton_wait": {"avg_ms": 68.407, "count": 124, "max_ms": 92.872, "p95_ms": 89.006}, "visualize": {"avg_ms": 10.833, "count": 124, "max_ms": 14.899, "p95_ms": 12.076}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.779, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3019}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4398.145, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T05:38:21+09:00", "last_error_summary": "[h264 @ 0x55ad7e1af100] error while decoding MB 22 38, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 236, "h264_decode_error_count": 418, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:38:21+09:00", "last_error_summary": "[h264 @ 0x55ad7e1af100] error while decoding MB 22 38, bytestream -5", "last_exit_at": "2026-07-02T05:38:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.631, "last_started_at": "2026-07-02T05:36:31+09:00", "pid": 40918, "process": {"exists": false, "pid": 40918}, "restart_count": 262, "restart_count_per_hour": 27, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 263}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 255.109, "sample_count": 1221, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 362308, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:38:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T05:38:21+09:00", "last_frame_read_age_sec": 0.227, "last_frame_read_at": "2026-07-02T05:38:20+09:00", "mjpeg": {"age_sec": 0.216, "exists": true, "mtime": "2026-07-02T05:38:20.983105+09:00", "mtime_epoch": 1782938300.9831054, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175236}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:38:21+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1043.859, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 125, "pushed_frames": 125, "read_frames": 124, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.173, "pushed": 4.173, "read": 4.14}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.095, "count": 124, "max_ms": 14.653, "p95_ms": 10.501}, "gstreamer_push": {"avg_ms": 0.945, "count": 125, "max_ms": 2.859, "p95_ms": 1.767}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.011, "count": 124, "max_ms": 71.245, "p95_ms": 58.516}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.13, "count": 124, "max_ms": 92.862, "p95_ms": 85.602}, "yolo11_640_fire_11class": {"avg_ms": 52.198, "count": 124, "max_ms": 72.979, "p95_ms": 69.58}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.539, "count": 124, "max_ms": 92.851, "p95_ms": 87.604}}, "truncated_count": 0}, "processing_total": {"avg_ms": 231.896, "count": 125, "max_ms": 317.974, "p95_ms": 279.21}, "read_interval": {"avg_ms": 241.62, "count": 124, "max_ms": 328.429, "p95_ms": 290.666}, "triton_wait": {"avg_ms": 68.322, "count": 124, "max_ms": 92.872, "p95_ms": 89.006}, "visualize": {"avg_ms": 10.829, "count": 124, "max_ms": 14.899, "p95_ms": 12.076}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.954, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3019}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4398.137, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 236, "h264_decode_error_count": 418, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:38:21+09:00", "last_error_summary": "[h264 @ 0x55ad7e1af100] error while decoding MB 22 38, bytestream -5", "last_exit_at": "2026-07-02T05:38:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.635, "last_started_at": "2026-07-02T05:36:31+09:00", "pid": 40918, "process": {"exists": false, "pid": 40918}, "restart_count": 262, "restart_count_per_hour": 27, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 263}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 255.109, "sample_count": 1221, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 362308, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:38:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T05:38:21+09:00", "last_frame_read_age_sec": 0.23, "last_frame_read_at": "2026-07-02T05:38:20+09:00", "mjpeg": {"age_sec": 0.219, "exists": true, "mtime": "2026-07-02T05:38:20.983105+09:00", "mtime_epoch": 1782938300.9831054, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175236}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:38:21+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1043.859, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 125, "pushed_frames": 125, "read_frames": 124, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.173, "pushed": 4.173, "read": 4.139}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.095, "count": 124, "max_ms": 14.653, "p95_ms": 10.501}, "gstreamer_push": {"avg_ms": 0.945, "count": 125, "max_ms": 2.859, "p95_ms": 1.767}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.011, "count": 124, "max_ms": 71.245, "p95_ms": 58.516}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.13, "count": 124, "max_ms": 92.862, "p95_ms": 85.602}, "yolo11_640_fire_11class": {"avg_ms": 52.198, "count": 124, "max_ms": 72.979, "p95_ms": 69.58}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.539, "count": 124, "max_ms": 92.851, "p95_ms": 87.604}}, "truncated_count": 0}, "processing_total": {"avg_ms": 231.896, "count": 125, "max_ms": 317.974, "p95_ms": 279.21}, "read_interval": {"avg_ms": 241.62, "count": 124, "max_ms": 328.429, "p95_ms": 290.666}, "triton_wait": {"avg_ms": 68.322, "count": 124, "max_ms": 92.872, "p95_ms": 89.006}, "visualize": {"avg_ms": 10.829, "count": 124, "max_ms": 14.899, "p95_ms": 12.076}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.958, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3019}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4505.781, "limit_mb": 8796093022207.996}, "detail": {"pid": 41025, "restart_count": 263, "start_count": 264}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 236, "h264_decode_error_count": 418, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:38:21+09:00", "last_error_summary": "[h264 @ 0x55ad7e1af100] error while decoding MB 22 38, bytestream -5", "last_exit_at": "2026-07-02T05:38:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T05:38:23+09:00", "pid": 41025, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.246, "name": "ffmpeg", "pid": 41025, "rss_mb": 139.246, "state": "S (sleeping)", "threads": 106}, "restart_count": 263, "restart_count_per_hour": 28, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 264}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 255.109, "sample_count": 1204, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 362308, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:38:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.069, "last_frame_pushed_at": "2026-07-02T05:38:21+09:00", "last_frame_read_age_sec": 4.293, "last_frame_read_at": "2026-07-02T05:38:20+09:00", "mjpeg": {"age_sec": 4.049, "exists": true, "mtime": "2026-07-02T05:38:21.215109+09:00", "mtime_epoch": 1782938301.215109, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175732}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:38:25+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1043.859, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 108, "pushed_frames": 108, "read_frames": 107, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.614, "pushed": 3.614, "read": 3.581}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.139, "count": 107, "max_ms": 14.653, "p95_ms": 10.56}, "gstreamer_push": {"avg_ms": 0.95, "count": 108, "max_ms": 2.859, "p95_ms": 1.767}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.878, "count": 107, "max_ms": 71.245, "p95_ms": 58.73}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.008, "count": 107, "max_ms": 92.862, "p95_ms": 86.701}, "yolo11_640_fire_11class": {"avg_ms": 52.209, "count": 107, "max_ms": 72.979, "p95_ms": 71.64}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.551, "count": 107, "max_ms": 92.851, "p95_ms": 87.828}}, "truncated_count": 0}, "processing_total": {"avg_ms": 231.514, "count": 108, "max_ms": 317.974, "p95_ms": 279.21}, "read_interval": {"avg_ms": 241.276, "count": 107, "max_ms": 328.429, "p95_ms": 290.666}, "triton_wait": {"avg_ms": 68.222, "count": 107, "max_ms": 92.872, "p95_ms": 89.228}, "visualize": {"avg_ms": 10.841, "count": 107, "max_ms": 14.899, "p95_ms": 12.098}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.881, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3022}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4401.828, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T05:38:21+09:00", "last_error_summary": "[h264 @ 0x55ad7e1af100] error while decoding MB 22 38, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 236, "h264_decode_error_count": 418, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:38:21+09:00", "last_error_summary": "[h264 @ 0x55ad7e1af100] error while decoding MB 22 38, bytestream -5", "last_exit_at": "2026-07-02T05:40:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.626, "last_started_at": "2026-07-02T05:38:23+09:00", "pid": 41025, "process": {"exists": false, "pid": 41025}, "restart_count": 263, "restart_count_per_hour": 28, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 264}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 256.709, "sample_count": 1208, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 362746, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:40:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T05:40:11+09:00", "last_frame_read_age_sec": 0.232, "last_frame_read_at": "2026-07-02T05:40:11+09:00", "mjpeg": {"age_sec": 0.225, "exists": true, "mtime": "2026-07-02T05:40:11.664968+09:00", "mtime_epoch": 1782938411.6649678, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175856}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:40:11+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1044.457, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.146, "pushed": 4.146, "read": 4.112}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.093, "count": 123, "max_ms": 11.079, "p95_ms": 9.673}, "gstreamer_push": {"avg_ms": 0.928, "count": 124, "max_ms": 2.936, "p95_ms": 0.95}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.257, "count": 123, "max_ms": 65.308, "p95_ms": 60.362}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.338, "count": 123, "max_ms": 95.559, "p95_ms": 84.863}, "yolo11_640_fire_11class": {"avg_ms": 53.342, "count": 123, "max_ms": 75.867, "p95_ms": 70.234}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.99, "count": 123, "max_ms": 94.225, "p95_ms": 86.144}}, "truncated_count": 0}, "processing_total": {"avg_ms": 233.655, "count": 124, "max_ms": 303.036, "p95_ms": 256.847}, "read_interval": {"avg_ms": 243.323, "count": 123, "max_ms": 312.969, "p95_ms": 266.368}, "triton_wait": {"avg_ms": 68.452, "count": 123, "max_ms": 95.571, "p95_ms": 87.996}, "visualize": {"avg_ms": 10.932, "count": 123, "max_ms": 13.227, "p95_ms": 11.964}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.909, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3022}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4401.938, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 236, "h264_decode_error_count": 418, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:38:21+09:00", "last_error_summary": "[h264 @ 0x55ad7e1af100] error while decoding MB 22 38, bytestream -5", "last_exit_at": "2026-07-02T05:40:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.63, "last_started_at": "2026-07-02T05:38:23+09:00", "pid": 41025, "process": {"exists": false, "pid": 41025}, "restart_count": 263, "restart_count_per_hour": 28, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 264}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 256.709, "sample_count": 1208, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 362746, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:40:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T05:40:11+09:00", "last_frame_read_age_sec": 0.236, "last_frame_read_at": "2026-07-02T05:40:11+09:00", "mjpeg": {"age_sec": 0.228, "exists": true, "mtime": "2026-07-02T05:40:11.664968+09:00", "mtime_epoch": 1782938411.6649678, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175856}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:40:11+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1044.457, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.145, "pushed": 4.145, "read": 4.112}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.093, "count": 123, "max_ms": 11.079, "p95_ms": 9.673}, "gstreamer_push": {"avg_ms": 0.928, "count": 124, "max_ms": 2.936, "p95_ms": 0.95}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.257, "count": 123, "max_ms": 65.308, "p95_ms": 60.362}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.338, "count": 123, "max_ms": 95.559, "p95_ms": 84.863}, "yolo11_640_fire_11class": {"avg_ms": 53.342, "count": 123, "max_ms": 75.867, "p95_ms": 70.234}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.99, "count": 123, "max_ms": 94.225, "p95_ms": 86.144}}, "truncated_count": 0}, "processing_total": {"avg_ms": 233.655, "count": 124, "max_ms": 303.036, "p95_ms": 256.847}, "read_interval": {"avg_ms": 243.323, "count": 123, "max_ms": 312.969, "p95_ms": 266.368}, "triton_wait": {"avg_ms": 68.452, "count": 123, "max_ms": 95.571, "p95_ms": 87.996}, "visualize": {"avg_ms": 10.932, "count": 123, "max_ms": 13.227, "p95_ms": 11.964}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.913, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3022}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4510.016, "limit_mb": 8796093022207.996}, "detail": {"pid": 41132, "restart_count": 264, "start_count": 265}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 236, "h264_decode_error_count": 418, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:38:21+09:00", "last_error_summary": "[h264 @ 0x55ad7e1af100] error while decoding MB 22 38, bytestream -5", "last_exit_at": "2026-07-02T05:40:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T05:40:14+09:00", "pid": 41132, "process": {"exists": true, "fd_count": 5, "hwm_mb": 144.742, "name": "ffmpeg", "pid": 41132, "rss_mb": 144.742, "state": "S (sleeping)", "threads": 106}, "restart_count": 264, "restart_count_per_hour": 29, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 265}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 256.669, "sample_count": 1191, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 362746, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:40:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.16, "last_frame_pushed_at": "2026-07-02T05:40:11+09:00", "last_frame_read_age_sec": 4.389, "last_frame_read_at": "2026-07-02T05:40:11+09:00", "mjpeg": {"age_sec": 4.142, "exists": true, "mtime": "2026-07-02T05:40:11.904972+09:00", "mtime_epoch": 1782938411.9049718, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 176465}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:40:16+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1044.457, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 107, "pushed_frames": 107, "read_frames": 106, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.576, "pushed": 3.576, "read": 3.543}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.068, "count": 106, "max_ms": 11.079, "p95_ms": 9.673}, "gstreamer_push": {"avg_ms": 0.927, "count": 107, "max_ms": 2.936, "p95_ms": 0.948}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.507, "count": 106, "max_ms": 65.308, "p95_ms": 60.362}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.682, "count": 106, "max_ms": 95.559, "p95_ms": 86.526}, "yolo11_640_fire_11class": {"avg_ms": 53.636, "count": 106, "max_ms": 75.867, "p95_ms": 70.564}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.24, "count": 106, "max_ms": 94.225, "p95_ms": 86.144}}, "truncated_count": 0}, "processing_total": {"avg_ms": 233.276, "count": 107, "max_ms": 303.036, "p95_ms": 256.847}, "read_interval": {"avg_ms": 242.94, "count": 106, "max_ms": 312.969, "p95_ms": 266.368}, "triton_wait": {"avg_ms": 68.78, "count": 106, "max_ms": 95.571, "p95_ms": 88.461}, "visualize": {"avg_ms": 10.951, "count": 106, "max_ms": 13.182, "p95_ms": 11.964}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.92, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3026}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4405.918, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55b6d39f53c0] error while decoding MB 13 22, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 237, "h264_decode_error_count": 419, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:42:08+09:00", "last_error_summary": "[h264 @ 0x55b6d39f53c0] error while decoding MB 13 22, bytestream -7", "last_exit_at": "2026-07-02T05:40:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 114.203, "last_started_at": "2026-07-02T05:40:14+09:00", "pid": 41132, "process": {"exists": false, "pid": 41132}, "restart_count": 264, "restart_count_per_hour": 28, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 265}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 281.463, "sample_count": 1158, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 363160, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:42:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.065, "last_frame_pushed_at": "2026-07-02T05:42:08+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-02T05:42:08+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-02T05:42:08.202929+09:00", "mtime_epoch": 1782938528.202929, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180615}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:42:08+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1047.379, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 113, "pushed_frames": 113, "read_frames": 113, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.779, "pushed": 3.779, "read": 3.779}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.258, "count": 112, "max_ms": 15.735, "p95_ms": 11.866}, "gstreamer_push": {"avg_ms": 0.911, "count": 113, "max_ms": 1.863, "p95_ms": 1.213}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.534, "count": 112, "max_ms": 74.679, "p95_ms": 68.501}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.863, "count": 112, "max_ms": 95.389, "p95_ms": 93.941}, "yolo11_640_fire_11class": {"avg_ms": 56.886, "count": 112, "max_ms": 75.281, "p95_ms": 73.954}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.345, "count": 112, "max_ms": 97.092, "p95_ms": 94.128}}, "truncated_count": 0}, "processing_total": {"avg_ms": 256.414, "count": 113, "max_ms": 351.709, "p95_ms": 283.406}, "read_interval": {"avg_ms": 266.178, "count": 113, "max_ms": 362.831, "p95_ms": 292.329}, "triton_wait": {"avg_ms": 76.394, "count": 112, "max_ms": 97.103, "p95_ms": 94.975}, "visualize": {"avg_ms": 11.546, "count": 112, "max_ms": 15.265, "p95_ms": 13.456}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.904, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3026}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4405.145, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T05:42:08+09:00", "last_error_summary": "[h264 @ 0x55b6d39f53c0] error while decoding MB 13 22, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 237, "h264_decode_error_count": 419, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:42:08+09:00", "last_error_summary": "[h264 @ 0x55b6d39f53c0] error while decoding MB 13 22, bytestream -7", "last_exit_at": "2026-07-02T05:42:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 114.409, "last_started_at": "2026-07-02T05:40:14+09:00", "pid": 41132, "process": {"exists": false, "pid": 41132}, "restart_count": 264, "restart_count_per_hour": 28, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 265}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 281.463, "sample_count": 1158, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 363161, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:42:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T05:42:08+09:00", "last_frame_read_age_sec": 0.261, "last_frame_read_at": "2026-07-02T05:42:08+09:00", "mjpeg": {"age_sec": 0.252, "exists": true, "mtime": "2026-07-02T05:42:08.202929+09:00", "mtime_epoch": 1782938528.202929, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180615}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:42:08+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1047.379, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 113, "pushed_frames": 113, "read_frames": 112, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.785, "pushed": 3.785, "read": 3.751}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.257, "count": 113, "max_ms": 15.735, "p95_ms": 11.866}, "gstreamer_push": {"avg_ms": 0.911, "count": 113, "max_ms": 1.863, "p95_ms": 1.213}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.578, "count": 113, "max_ms": 74.679, "p95_ms": 68.501}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.842, "count": 113, "max_ms": 95.389, "p95_ms": 93.941}, "yolo11_640_fire_11class": {"avg_ms": 56.863, "count": 113, "max_ms": 75.281, "p95_ms": 73.954}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.297, "count": 113, "max_ms": 97.092, "p95_ms": 94.128}}, "truncated_count": 0}, "processing_total": {"avg_ms": 256.657, "count": 113, "max_ms": 351.709, "p95_ms": 283.406}, "read_interval": {"avg_ms": 266.423, "count": 112, "max_ms": 362.831, "p95_ms": 292.329}, "triton_wait": {"avg_ms": 76.369, "count": 113, "max_ms": 97.103, "p95_ms": 94.975}, "visualize": {"avg_ms": 11.548, "count": 113, "max_ms": 15.265, "p95_ms": 13.456}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.856, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3026}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4405.137, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 237, "h264_decode_error_count": 419, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:42:08+09:00", "last_error_summary": "[h264 @ 0x55b6d39f53c0] error while decoding MB 13 22, bytestream -7", "last_exit_at": "2026-07-02T05:42:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 114.412, "last_started_at": "2026-07-02T05:40:14+09:00", "pid": 41132, "process": {"exists": false, "pid": 41132}, "restart_count": 264, "restart_count_per_hour": 28, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 265}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 281.463, "sample_count": 1158, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 363161, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:42:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T05:42:08+09:00", "last_frame_read_age_sec": 0.265, "last_frame_read_at": "2026-07-02T05:42:08+09:00", "mjpeg": {"age_sec": 0.255, "exists": true, "mtime": "2026-07-02T05:42:08.202929+09:00", "mtime_epoch": 1782938528.202929, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180615}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:42:08+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1047.379, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 113, "pushed_frames": 113, "read_frames": 112, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.784, "pushed": 3.784, "read": 3.751}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.264, "count": 112, "max_ms": 15.735, "p95_ms": 11.866}, "gstreamer_push": {"avg_ms": 0.911, "count": 113, "max_ms": 1.863, "p95_ms": 1.213}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.495, "count": 112, "max_ms": 74.679, "p95_ms": 68.501}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.801, "count": 112, "max_ms": 95.389, "p95_ms": 93.941}, "yolo11_640_fire_11class": {"avg_ms": 56.797, "count": 112, "max_ms": 75.281, "p95_ms": 73.954}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.257, "count": 112, "max_ms": 97.092, "p95_ms": 94.128}}, "truncated_count": 0}, "processing_total": {"avg_ms": 256.657, "count": 113, "max_ms": 351.709, "p95_ms": 283.406}, "read_interval": {"avg_ms": 266.423, "count": 112, "max_ms": 362.831, "p95_ms": 292.329}, "triton_wait": {"avg_ms": 76.332, "count": 112, "max_ms": 97.103, "p95_ms": 94.975}, "visualize": {"avg_ms": 11.554, "count": 112, "max_ms": 15.265, "p95_ms": 13.456}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.86, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3026}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4515.816, "limit_mb": 8796093022207.996}, "detail": {"pid": 41239, "restart_count": 265, "start_count": 266}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 237, "h264_decode_error_count": 419, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:42:08+09:00", "last_error_summary": "[h264 @ 0x55b6d39f53c0] error while decoding MB 13 22, bytestream -7", "last_exit_at": "2026-07-02T05:42:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T05:42:11+09:00", "pid": 41239, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.758, "name": "ffmpeg", "pid": 41239, "rss_mb": 138.758, "state": "S (sleeping)", "threads": 106}, "restart_count": 265, "restart_count_per_hour": 29, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 266}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 281.937, "sample_count": 1136, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 363161, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:42:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.925, "last_frame_pushed_at": "2026-07-02T05:42:08+09:00", "last_frame_read_age_sec": 5.183, "last_frame_read_at": "2026-07-02T05:42:08+09:00", "mjpeg": {"age_sec": 4.906, "exists": true, "mtime": "2026-07-02T05:42:08.470933+09:00", "mtime_epoch": 1782938528.4709334, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180207}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:42:13+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1047.379, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 93, "pushed_frames": 93, "read_frames": 92, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.115, "pushed": 3.115, "read": 3.082}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.308, "count": 93, "max_ms": 15.735, "p95_ms": 11.866}, "gstreamer_push": {"avg_ms": 0.907, "count": 93, "max_ms": 1.863, "p95_ms": 1.213}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.861, "count": 93, "max_ms": 74.679, "p95_ms": 69.941}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.3, "count": 93, "max_ms": 95.389, "p95_ms": 94.031}, "yolo11_640_fire_11class": {"avg_ms": 58.034, "count": 93, "max_ms": 75.281, "p95_ms": 74.325}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.117, "count": 93, "max_ms": 97.092, "p95_ms": 94.962}}, "truncated_count": 0}, "processing_total": {"avg_ms": 261.106, "count": 93, "max_ms": 351.709, "p95_ms": 295.245}, "read_interval": {"avg_ms": 270.924, "count": 92, "max_ms": 362.831, "p95_ms": 306.308}, "triton_wait": {"avg_ms": 77.938, "count": 93, "max_ms": 97.103, "p95_ms": 95.082}, "visualize": {"avg_ms": 11.594, "count": 93, "max_ms": 15.265, "p95_ms": 13.011}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.852, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3029}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4404.41, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55c6ce2ab9c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c6ce2ab9c0] error while decoding MB 0 43, bytestream 6099 [h264 @ 0x55c6ce1df580] error while decoding MB 47 8, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 238, "h264_decode_error_count": 421, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:44:09+09:00", "last_error_summary": "[h264 @ 0x55c6ce2ab9c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c6ce2ab9c0] error while decoding MB 0 43, bytestream 6099 [h264 @ 0x55c6ce1df580] error while decoding MB 47 8, bytestream -5", "last_exit_at": "2026-07-02T05:42:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 118.387, "last_started_at": "2026-07-02T05:42:11+09:00", "pid": 41239, "process": {"exists": false, "pid": 41239}, "restart_count": 265, "restart_count_per_hour": 29, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 266}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 289.765, "sample_count": 1101, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 363591, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:44:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-02T05:44:09+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-02T05:44:09+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-02T05:44:09.723330+09:00", "mtime_epoch": 1782938649.7233305, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179622}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:44:09+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1044.641, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 111, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.729, "pushed": 3.729, "read": 3.729}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.183, "count": 111, "max_ms": 11.65, "p95_ms": 11.015}, "gstreamer_push": {"avg_ms": 0.909, "count": 111, "max_ms": 2.971, "p95_ms": 1.028}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.569, "count": 111, "max_ms": 69.34, "p95_ms": 68.528}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.105, "count": 111, "max_ms": 100.235, "p95_ms": 93.159}, "yolo11_640_fire_11class": {"avg_ms": 56.815, "count": 111, "max_ms": 74.143, "p95_ms": 72.776}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.372, "count": 111, "max_ms": 99.419, "p95_ms": 93.145}}, "truncated_count": 0}, "processing_total": {"avg_ms": 260.326, "count": 111, "max_ms": 338.708, "p95_ms": 280.156}, "read_interval": {"avg_ms": 269.901, "count": 111, "max_ms": 348.012, "p95_ms": 290.539}, "triton_wait": {"avg_ms": 75.609, "count": 111, "max_ms": 100.247, "p95_ms": 93.512}, "visualize": {"avg_ms": 11.831, "count": 111, "max_ms": 18.429, "p95_ms": 12.511}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.764, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3029}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4403.293, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T05:44:09+09:00", "last_error_summary": "[h264 @ 0x55c6ce2ab9c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c6ce2ab9c0] error while decoding MB 0 43, bytestream 6099 [h264 @ 0x55c6ce1df580] error while decoding MB 47 8, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 238, "h264_decode_error_count": 421, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:44:09+09:00", "last_error_summary": "[h264 @ 0x55c6ce2ab9c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c6ce2ab9c0] error while decoding MB 0 43, bytestream 6099 [h264 @ 0x55c6ce1df580] error while decoding MB 47 8, bytestream -5", "last_exit_at": "2026-07-02T05:44:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 118.622, "last_started_at": "2026-07-02T05:42:11+09:00", "pid": 41239, "process": {"exists": false, "pid": 41239}, "restart_count": 265, "restart_count_per_hour": 29, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 266}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 289.765, "sample_count": 1101, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 363592, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:44:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T05:44:09+09:00", "last_frame_read_age_sec": 0.285, "last_frame_read_at": "2026-07-02T05:44:09+09:00", "mjpeg": {"age_sec": 0.275, "exists": true, "mtime": "2026-07-02T05:44:09.723330+09:00", "mtime_epoch": 1782938649.7233305, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179622}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:44:09+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1044.641, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 112, "pushed_frames": 112, "read_frames": 111, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.733, "pushed": 3.733, "read": 3.7}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.193, "count": 111, "max_ms": 11.65, "p95_ms": 11.015}, "gstreamer_push": {"avg_ms": 0.908, "count": 112, "max_ms": 2.971, "p95_ms": 1.028}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.842, "count": 111, "max_ms": 75.803, "p95_ms": 68.723}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.44, "count": 111, "max_ms": 100.235, "p95_ms": 93.499}, "yolo11_640_fire_11class": {"avg_ms": 57.1, "count": 111, "max_ms": 77.716, "p95_ms": 73.037}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.745, "count": 111, "max_ms": 101.056, "p95_ms": 93.196}}, "truncated_count": 0}, "processing_total": {"avg_ms": 260.513, "count": 112, "max_ms": 338.708, "p95_ms": 281.298}, "read_interval": {"avg_ms": 269.901, "count": 111, "max_ms": 348.012, "p95_ms": 290.539}, "triton_wait": {"avg_ms": 75.954, "count": 111, "max_ms": 101.068, "p95_ms": 93.603}, "visualize": {"avg_ms": 11.825, "count": 111, "max_ms": 18.429, "p95_ms": 12.511}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3029}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4403.285, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 238, "h264_decode_error_count": 421, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:44:09+09:00", "last_error_summary": "[h264 @ 0x55c6ce2ab9c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c6ce2ab9c0] error while decoding MB 0 43, bytestream 6099 [h264 @ 0x55c6ce1df580] error while decoding MB 47 8, bytestream -5", "last_exit_at": "2026-07-02T05:44:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 118.625, "last_started_at": "2026-07-02T05:42:11+09:00", "pid": 41239, "process": {"exists": false, "pid": 41239}, "restart_count": 265, "restart_count_per_hour": 29, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 266}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 289.765, "sample_count": 1101, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 363592, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:44:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T05:44:09+09:00", "last_frame_read_age_sec": 0.288, "last_frame_read_at": "2026-07-02T05:44:09+09:00", "mjpeg": {"age_sec": 0.278, "exists": true, "mtime": "2026-07-02T05:44:09.723330+09:00", "mtime_epoch": 1782938649.7233305, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179622}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:44:10+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1044.641, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 111, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.701, "pushed": 3.701, "read": 3.701}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.193, "count": 111, "max_ms": 11.65, "p95_ms": 11.015}, "gstreamer_push": {"avg_ms": 0.909, "count": 111, "max_ms": 2.971, "p95_ms": 1.028}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.842, "count": 111, "max_ms": 75.803, "p95_ms": 68.723}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.44, "count": 111, "max_ms": 100.235, "p95_ms": 93.499}, "yolo11_640_fire_11class": {"avg_ms": 57.1, "count": 111, "max_ms": 77.716, "p95_ms": 73.037}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.745, "count": 111, "max_ms": 101.056, "p95_ms": 93.196}}, "truncated_count": 0}, "processing_total": {"avg_ms": 260.655, "count": 111, "max_ms": 338.708, "p95_ms": 281.298}, "read_interval": {"avg_ms": 269.901, "count": 111, "max_ms": 348.012, "p95_ms": 290.539}, "triton_wait": {"avg_ms": 75.954, "count": 111, "max_ms": 101.068, "p95_ms": 93.603}, "visualize": {"avg_ms": 11.825, "count": 111, "max_ms": 18.429, "p95_ms": 12.511}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3029}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4522.555, "limit_mb": 8796093022207.996}, "detail": {"pid": 41346, "restart_count": 266, "start_count": 267}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 238, "h264_decode_error_count": 421, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:44:09+09:00", "last_error_summary": "[h264 @ 0x55c6ce2ab9c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55c6ce2ab9c0] error while decoding MB 0 43, bytestream 6099 [h264 @ 0x55c6ce1df580] error while decoding MB 47 8, bytestream -5", "last_exit_at": "2026-07-02T05:44:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T05:44:12+09:00", "pid": 41346, "process": {"exists": true, "fd_count": 5, "hwm_mb": 147.152, "name": "ffmpeg", "pid": 41346, "rss_mb": 147.152, "state": "S (sleeping)", "threads": 106}, "restart_count": 266, "restart_count_per_hour": 30, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 267}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 289.874, "sample_count": 1084, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 363592, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:44:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.068, "last_frame_pushed_at": "2026-07-02T05:44:09+09:00", "last_frame_read_age_sec": 4.351, "last_frame_read_at": "2026-07-02T05:44:09+09:00", "mjpeg": {"age_sec": 4.052, "exists": true, "mtime": "2026-07-02T05:44:10.011343+09:00", "mtime_epoch": 1782938650.0113432, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179770}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:44:14+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1044.641, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 95, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.207, "pushed": 3.207, "read": 3.173}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.323, "count": 95, "max_ms": 11.65, "p95_ms": 11.341}, "gstreamer_push": {"avg_ms": 0.916, "count": 96, "max_ms": 2.971, "p95_ms": 1.741}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.578, "count": 95, "max_ms": 75.803, "p95_ms": 68.726}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.508, "count": 95, "max_ms": 100.235, "p95_ms": 93.509}, "yolo11_640_fire_11class": {"avg_ms": 57.989, "count": 95, "max_ms": 77.716, "p95_ms": 73.593}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.837, "count": 95, "max_ms": 101.056, "p95_ms": 94.224}}, "truncated_count": 0}, "processing_total": {"avg_ms": 262.566, "count": 96, "max_ms": 338.708, "p95_ms": 285.193}, "read_interval": {"avg_ms": 271.972, "count": 95, "max_ms": 348.012, "p95_ms": 294.178}, "triton_wait": {"avg_ms": 77.063, "count": 95, "max_ms": 101.068, "p95_ms": 94.236}, "visualize": {"avg_ms": 11.818, "count": 95, "max_ms": 13.624, "p95_ms": 12.387}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.938, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3033}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4404.938, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x557eace4f400] error while decoding MB 34 23, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 239, "h264_decode_error_count": 422, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:46:13+09:00", "last_error_summary": "[h264 @ 0x557eace4f400] error while decoding MB 34 23, bytestream -9", "last_exit_at": "2026-07-02T05:44:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 120.997, "last_started_at": "2026-07-02T05:44:12+09:00", "pid": 41346, "process": {"exists": false, "pid": 41346}, "restart_count": 266, "restart_count_per_hour": 30, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 267}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 314.266, "sample_count": 1043, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 363996, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:46:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-02T05:46:12+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-02T05:46:13+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-02T05:46:13.012503+09:00", "mtime_epoch": 1782938773.012503, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173411}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:46:13+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1044.766, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 101, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.384, "pushed": 3.384, "read": 3.384}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 12.728, "count": 100, "max_ms": 17.762, "p95_ms": 15.878}, "gstreamer_push": {"avg_ms": 0.922, "count": 101, "max_ms": 1.964, "p95_ms": 1.772}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.55, "count": 100, "max_ms": 72.931, "p95_ms": 64.337}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.454, "count": 100, "max_ms": 100.487, "p95_ms": 93.68}, "yolo11_640_fire_11class": {"avg_ms": 54.803, "count": 100, "max_ms": 76.653, "p95_ms": 72.58}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.8, "count": 100, "max_ms": 101.194, "p95_ms": 92.905}}, "truncated_count": 0}, "processing_total": {"avg_ms": 288.386, "count": 101, "max_ms": 394.478, "p95_ms": 347.799}, "read_interval": {"avg_ms": 297.753, "count": 101, "max_ms": 404.603, "p95_ms": 358.679}, "triton_wait": {"avg_ms": 74.599, "count": 100, "max_ms": 101.205, "p95_ms": 94.109}, "visualize": {"avg_ms": 12.679, "count": 100, "max_ms": 17.734, "p95_ms": 14.388}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.843, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3033}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4403.707, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T05:46:13+09:00", "last_error_summary": "[h264 @ 0x557eace4f400] error while decoding MB 34 23, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 239, "h264_decode_error_count": 422, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:46:13+09:00", "last_error_summary": "[h264 @ 0x557eace4f400] error while decoding MB 34 23, bytestream -9", "last_exit_at": "2026-07-02T05:46:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.325, "last_started_at": "2026-07-02T05:44:12+09:00", "pid": 41346, "process": {"exists": false, "pid": 41346}, "restart_count": 266, "restart_count_per_hour": 30, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 267}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 314.266, "sample_count": 1042, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 363997, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:46:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T05:46:13+09:00", "last_frame_read_age_sec": 0.384, "last_frame_read_at": "2026-07-02T05:46:13+09:00", "mjpeg": {"age_sec": 0.376, "exists": true, "mtime": "2026-07-02T05:46:13.012503+09:00", "mtime_epoch": 1782938773.012503, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173411}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:46:13+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1044.766, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 100, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.381, "pushed": 3.381, "read": 3.347}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 12.751, "count": 100, "max_ms": 17.762, "p95_ms": 15.878}, "gstreamer_push": {"avg_ms": 0.925, "count": 101, "max_ms": 1.964, "p95_ms": 1.772}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.458, "count": 100, "max_ms": 72.931, "p95_ms": 64.337}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.332, "count": 100, "max_ms": 100.487, "p95_ms": 93.68}, "yolo11_640_fire_11class": {"avg_ms": 54.712, "count": 100, "max_ms": 76.653, "p95_ms": 72.58}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.728, "count": 100, "max_ms": 101.194, "p95_ms": 92.905}}, "truncated_count": 0}, "processing_total": {"avg_ms": 289.292, "count": 101, "max_ms": 394.478, "p95_ms": 356.159}, "read_interval": {"avg_ms": 297.774, "count": 100, "max_ms": 404.603, "p95_ms": 358.679}, "triton_wait": {"avg_ms": 74.492, "count": 100, "max_ms": 101.205, "p95_ms": 94.109}, "visualize": {"avg_ms": 12.726, "count": 100, "max_ms": 17.734, "p95_ms": 14.677}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.876, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3033}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4403.699, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 239, "h264_decode_error_count": 422, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:46:13+09:00", "last_error_summary": "[h264 @ 0x557eace4f400] error while decoding MB 34 23, bytestream -9", "last_exit_at": "2026-07-02T05:46:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.329, "last_started_at": "2026-07-02T05:44:12+09:00", "pid": 41346, "process": {"exists": false, "pid": 41346}, "restart_count": 266, "restart_count_per_hour": 30, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 267}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 314.266, "sample_count": 1042, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 363997, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:46:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-02T05:46:13+09:00", "last_frame_read_age_sec": 0.387, "last_frame_read_at": "2026-07-02T05:46:13+09:00", "mjpeg": {"age_sec": 0.379, "exists": true, "mtime": "2026-07-02T05:46:13.012503+09:00", "mtime_epoch": 1782938773.012503, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 173411}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:46:13+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1044.766, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 100, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.38, "pushed": 3.38, "read": 3.347}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 12.751, "count": 100, "max_ms": 17.762, "p95_ms": 15.878}, "gstreamer_push": {"avg_ms": 0.925, "count": 101, "max_ms": 1.964, "p95_ms": 1.772}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.458, "count": 100, "max_ms": 72.931, "p95_ms": 64.337}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.332, "count": 100, "max_ms": 100.487, "p95_ms": 93.68}, "yolo11_640_fire_11class": {"avg_ms": 54.712, "count": 100, "max_ms": 76.653, "p95_ms": 72.58}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.728, "count": 100, "max_ms": 101.194, "p95_ms": 92.905}}, "truncated_count": 0}, "processing_total": {"avg_ms": 289.292, "count": 101, "max_ms": 394.478, "p95_ms": 356.159}, "read_interval": {"avg_ms": 297.774, "count": 100, "max_ms": 404.603, "p95_ms": 358.679}, "triton_wait": {"avg_ms": 74.492, "count": 100, "max_ms": 101.205, "p95_ms": 94.109}, "visualize": {"avg_ms": 12.726, "count": 100, "max_ms": 17.734, "p95_ms": 14.677}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.879, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3033}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4515.148, "limit_mb": 8796093022207.996}, "detail": {"pid": 41453, "restart_count": 267, "start_count": 268}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 239, "h264_decode_error_count": 422, "h264_decode_error_count_per_hour": 33, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:46:13+09:00", "last_error_summary": "[h264 @ 0x557eace4f400] error while decoding MB 34 23, bytestream -9", "last_exit_at": "2026-07-02T05:46:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T05:46:15+09:00", "pid": 41453, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.441, "name": "ffmpeg", "pid": 41453, "rss_mb": 138.441, "state": "S (sleeping)", "threads": 106}, "restart_count": 267, "restart_count_per_hour": 31, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 268}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 314.294, "sample_count": 1026, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 363997, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:46:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.484, "last_frame_pushed_at": "2026-07-02T05:46:13+09:00", "last_frame_read_age_sec": 4.863, "last_frame_read_at": "2026-07-02T05:46:13+09:00", "mjpeg": {"age_sec": 4.467, "exists": true, "mtime": "2026-07-02T05:46:13.400518+09:00", "mtime_epoch": 1782938773.4005182, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172829}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:46:17+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1044.766, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 85, "pushed_frames": 85, "read_frames": 84, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.844, "pushed": 2.844, "read": 2.81}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 12.88, "count": 84, "max_ms": 17.762, "p95_ms": 15.916}, "gstreamer_push": {"avg_ms": 0.915, "count": 85, "max_ms": 1.964, "p95_ms": 1.766}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.368, "count": 84, "max_ms": 72.931, "p95_ms": 66.719}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.483, "count": 84, "max_ms": 100.487, "p95_ms": 94.14}, "yolo11_640_fire_11class": {"avg_ms": 55.145, "count": 84, "max_ms": 76.653, "p95_ms": 73.502}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.234, "count": 84, "max_ms": 101.194, "p95_ms": 94.096}}, "truncated_count": 0}, "processing_total": {"avg_ms": 292.948, "count": 85, "max_ms": 394.478, "p95_ms": 363.0}, "read_interval": {"avg_ms": 301.247, "count": 84, "max_ms": 404.603, "p95_ms": 366.046}, "triton_wait": {"avg_ms": 74.671, "count": 84, "max_ms": 101.205, "p95_ms": 94.153}, "visualize": {"avg_ms": 12.901, "count": 84, "max_ms": 17.734, "p95_ms": 14.834}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.892, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3036}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4410.125, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x562e9f46efc0] error while decoding MB 6 18, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 240, "h264_decode_error_count": 423, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:48:17+09:00", "last_error_summary": "[h264 @ 0x562e9f46efc0] error while decoding MB 6 18, bytestream -7", "last_exit_at": "2026-07-02T05:46:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.251, "last_started_at": "2026-07-02T05:46:15+09:00", "pid": 41453, "process": {"exists": false, "pid": 41453}, "restart_count": 267, "restart_count_per_hour": 30, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 268}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 327.432, "sample_count": 997, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 364393, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:48:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-02T05:48:17+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-02T05:48:17+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-02T05:48:17.077171+09:00", "mtime_epoch": 1782938897.0771706, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182149}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:48:17+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1048.172, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 101, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.333, "pushed": 3.333, "read": 3.367}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 12.594, "count": 100, "max_ms": 20.299, "p95_ms": 16.974}, "gstreamer_push": {"avg_ms": 0.974, "count": 100, "max_ms": 3.024, "p95_ms": 1.914}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.36, "count": 100, "max_ms": 65.458, "p95_ms": 61.633}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.473, "count": 100, "max_ms": 96.958, "p95_ms": 93.058}, "yolo11_640_fire_11class": {"avg_ms": 54.608, "count": 100, "max_ms": 73.114, "p95_ms": 71.163}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.159, "count": 100, "max_ms": 96.219, "p95_ms": 93.478}}, "truncated_count": 0}, "processing_total": {"avg_ms": 289.813, "count": 100, "max_ms": 416.125, "p95_ms": 315.978}, "read_interval": {"avg_ms": 299.188, "count": 101, "max_ms": 428.624, "p95_ms": 325.392}, "triton_wait": {"avg_ms": 75.4, "count": 100, "max_ms": 96.969, "p95_ms": 93.489}, "visualize": {"avg_ms": 13.274, "count": 100, "max_ms": 19.25, "p95_ms": 15.467}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3036}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4408.789, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T05:48:17+09:00", "last_error_summary": "[h264 @ 0x562e9f46efc0] error while decoding MB 6 18, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 240, "h264_decode_error_count": 423, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:48:17+09:00", "last_error_summary": "[h264 @ 0x562e9f46efc0] error while decoding MB 6 18, bytestream -7", "last_exit_at": "2026-07-02T05:48:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.504, "last_started_at": "2026-07-02T05:46:15+09:00", "pid": 41453, "process": {"exists": false, "pid": 41453}, "restart_count": 267, "restart_count_per_hour": 30, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 268}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 327.432, "sample_count": 997, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 364394, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:48:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T05:48:17+09:00", "last_frame_read_age_sec": 0.303, "last_frame_read_at": "2026-07-02T05:48:17+09:00", "mjpeg": {"age_sec": 0.293, "exists": true, "mtime": "2026-07-02T05:48:17.077171+09:00", "mtime_epoch": 1782938897.0771706, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182149}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:48:17+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1048.172, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 100, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.372, "pushed": 3.372, "read": 3.339}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 12.601, "count": 100, "max_ms": 20.299, "p95_ms": 16.974}, "gstreamer_push": {"avg_ms": 0.973, "count": 101, "max_ms": 3.024, "p95_ms": 1.914}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.538, "count": 100, "max_ms": 65.458, "p95_ms": 62.115}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.643, "count": 100, "max_ms": 96.958, "p95_ms": 93.247}, "yolo11_640_fire_11class": {"avg_ms": 54.794, "count": 100, "max_ms": 73.114, "p95_ms": 71.401}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.392, "count": 100, "max_ms": 96.219, "p95_ms": 93.754}}, "truncated_count": 0}, "processing_total": {"avg_ms": 289.896, "count": 101, "max_ms": 416.125, "p95_ms": 315.978}, "read_interval": {"avg_ms": 299.487, "count": 100, "max_ms": 428.624, "p95_ms": 325.392}, "triton_wait": {"avg_ms": 75.591, "count": 100, "max_ms": 96.969, "p95_ms": 94.043}, "visualize": {"avg_ms": 13.283, "count": 100, "max_ms": 19.25, "p95_ms": 15.467}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.953, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3036}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4408.781, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 240, "h264_decode_error_count": 423, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:48:17+09:00", "last_error_summary": "[h264 @ 0x562e9f46efc0] error while decoding MB 6 18, bytestream -7", "last_exit_at": "2026-07-02T05:48:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.507, "last_started_at": "2026-07-02T05:46:15+09:00", "pid": 41453, "process": {"exists": false, "pid": 41453}, "restart_count": 267, "restart_count_per_hour": 30, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 268}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 327.432, "sample_count": 997, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 364394, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:48:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T05:48:17+09:00", "last_frame_read_age_sec": 0.306, "last_frame_read_at": "2026-07-02T05:48:17+09:00", "mjpeg": {"age_sec": 0.296, "exists": true, "mtime": "2026-07-02T05:48:17.077171+09:00", "mtime_epoch": 1782938897.0771706, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182149}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:48:17+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1048.172, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 100, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.372, "pushed": 3.372, "read": 3.338}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 12.601, "count": 100, "max_ms": 20.299, "p95_ms": 16.974}, "gstreamer_push": {"avg_ms": 0.973, "count": 101, "max_ms": 3.024, "p95_ms": 1.914}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.538, "count": 100, "max_ms": 65.458, "p95_ms": 62.115}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.643, "count": 100, "max_ms": 96.958, "p95_ms": 93.247}, "yolo11_640_fire_11class": {"avg_ms": 54.794, "count": 100, "max_ms": 73.114, "p95_ms": 71.401}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.392, "count": 100, "max_ms": 96.219, "p95_ms": 93.754}}, "truncated_count": 0}, "processing_total": {"avg_ms": 289.896, "count": 101, "max_ms": 416.125, "p95_ms": 315.978}, "read_interval": {"avg_ms": 299.487, "count": 100, "max_ms": 428.624, "p95_ms": 325.392}, "triton_wait": {"avg_ms": 75.591, "count": 100, "max_ms": 96.969, "p95_ms": 94.043}, "visualize": {"avg_ms": 13.283, "count": 100, "max_ms": 19.25, "p95_ms": 15.467}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.956, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3036}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4516.383, "limit_mb": 8796093022207.996}, "detail": {"pid": 41560, "restart_count": 268, "start_count": 269}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 240, "h264_decode_error_count": 423, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:48:17+09:00", "last_error_summary": "[h264 @ 0x562e9f46efc0] error while decoding MB 6 18, bytestream -7", "last_exit_at": "2026-07-02T05:48:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T05:48:20+09:00", "pid": 41560, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.168, "name": "ffmpeg", "pid": 41560, "rss_mb": 139.168, "state": "S (sleeping)", "threads": 106}, "restart_count": 268, "restart_count_per_hour": 31, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 269}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 327.432, "sample_count": 978, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 364394, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:48:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.933, "last_frame_pushed_at": "2026-07-02T05:48:17+09:00", "last_frame_read_age_sec": 5.233, "last_frame_read_at": "2026-07-02T05:48:17+09:00", "mjpeg": {"age_sec": 4.918, "exists": true, "mtime": "2026-07-02T05:48:17.381181+09:00", "mtime_epoch": 1782938897.3811815, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182258}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:48:22+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1048.172, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 84, "pushed_frames": 84, "read_frames": 83, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.817, "pushed": 2.817, "read": 2.783}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 12.672, "count": 83, "max_ms": 20.299, "p95_ms": 17.359}, "gstreamer_push": {"avg_ms": 0.981, "count": 84, "max_ms": 3.024, "p95_ms": 1.914}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.3, "count": 83, "max_ms": 65.458, "p95_ms": 61.633}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.212, "count": 83, "max_ms": 96.958, "p95_ms": 93.247}, "yolo11_640_fire_11class": {"avg_ms": 54.653, "count": 83, "max_ms": 73.114, "p95_ms": 71.402}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.403, "count": 83, "max_ms": 96.219, "p95_ms": 94.032}}, "truncated_count": 0}, "processing_total": {"avg_ms": 290.152, "count": 84, "max_ms": 416.125, "p95_ms": 315.978}, "read_interval": {"avg_ms": 299.735, "count": 83, "max_ms": 428.624, "p95_ms": 325.392}, "triton_wait": {"avg_ms": 75.33, "count": 83, "max_ms": 96.969, "p95_ms": 95.005}, "visualize": {"avg_ms": 13.366, "count": 83, "max_ms": 19.25, "p95_ms": 17.82}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.823, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3040}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4407.691, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5650e45a5bc0] error while decoding MB 45 14, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 241, "h264_decode_error_count": 424, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:50:18+09:00", "last_error_summary": "[h264 @ 0x5650e45a5bc0] error while decoding MB 45 14, bytestream -5", "last_exit_at": "2026-07-02T05:48:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 118.133, "last_started_at": "2026-07-02T05:48:20+09:00", "pid": 41560, "process": {"exists": false, "pid": 41560}, "restart_count": 268, "restart_count_per_hour": 30, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 269}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 325.414, "sample_count": 994, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 364806, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:50:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-02T05:50:18+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-02T05:50:18+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-02T05:50:18.389325+09:00", "mtime_epoch": 1782939018.3893254, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182122}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:50:18+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1045.457, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 109, "pushed_frames": 109, "read_frames": 109, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.664, "pushed": 3.664, "read": 3.664}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.872, "count": 109, "max_ms": 12.996, "p95_ms": 12.098}, "gstreamer_push": {"avg_ms": 0.895, "count": 109, "max_ms": 2.868, "p95_ms": 0.908}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.714, "count": 109, "max_ms": 69.607, "p95_ms": 63.132}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.441, "count": 109, "max_ms": 96.668, "p95_ms": 94.665}, "yolo11_640_fire_11class": {"avg_ms": 53.526, "count": 109, "max_ms": 72.912, "p95_ms": 71.355}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.639, "count": 109, "max_ms": 99.156, "p95_ms": 94.166}}, "truncated_count": 0}, "processing_total": {"avg_ms": 266.446, "count": 109, "max_ms": 374.646, "p95_ms": 296.263}, "read_interval": {"avg_ms": 274.845, "count": 109, "max_ms": 383.251, "p95_ms": 305.322}, "triton_wait": {"avg_ms": 75.206, "count": 109, "max_ms": 99.169, "p95_ms": 95.148}, "visualize": {"avg_ms": 12.057, "count": 109, "max_ms": 14.855, "p95_ms": 12.933}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.752, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3040}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4407.074, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T05:50:18+09:00", "last_error_summary": "[h264 @ 0x5650e45a5bc0] error while decoding MB 45 14, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 241, "h264_decode_error_count": 424, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:50:18+09:00", "last_error_summary": "[h264 @ 0x5650e45a5bc0] error while decoding MB 45 14, bytestream -5", "last_exit_at": "2026-07-02T05:50:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 118.344, "last_started_at": "2026-07-02T05:48:20+09:00", "pid": 41560, "process": {"exists": false, "pid": 41560}, "restart_count": 268, "restart_count_per_hour": 30, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 269}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 325.414, "sample_count": 994, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 364807, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:50:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T05:50:18+09:00", "last_frame_read_age_sec": 0.262, "last_frame_read_at": "2026-07-02T05:50:18+09:00", "mjpeg": {"age_sec": 0.253, "exists": true, "mtime": "2026-07-02T05:50:18.389325+09:00", "mtime_epoch": 1782939018.3893254, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182122}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:50:18+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1045.457, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 110, "pushed_frames": 110, "read_frames": 109, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.671, "pushed": 3.671, "read": 3.638}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.896, "count": 109, "max_ms": 12.996, "p95_ms": 12.098}, "gstreamer_push": {"avg_ms": 0.895, "count": 110, "max_ms": 2.868, "p95_ms": 0.908}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.543, "count": 109, "max_ms": 69.607, "p95_ms": 63.132}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.321, "count": 109, "max_ms": 96.668, "p95_ms": 94.665}, "yolo11_640_fire_11class": {"avg_ms": 53.387, "count": 109, "max_ms": 72.912, "p95_ms": 71.355}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.458, "count": 109, "max_ms": 99.156, "p95_ms": 94.166}}, "truncated_count": 0}, "processing_total": {"avg_ms": 266.352, "count": 110, "max_ms": 374.646, "p95_ms": 296.263}, "read_interval": {"avg_ms": 274.845, "count": 109, "max_ms": 383.251, "p95_ms": 305.322}, "triton_wait": {"avg_ms": 75.045, "count": 109, "max_ms": 99.169, "p95_ms": 95.148}, "visualize": {"avg_ms": 12.081, "count": 109, "max_ms": 14.855, "p95_ms": 12.933}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.962, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3040}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4407.066, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 241, "h264_decode_error_count": 424, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:50:18+09:00", "last_error_summary": "[h264 @ 0x5650e45a5bc0] error while decoding MB 45 14, bytestream -5", "last_exit_at": "2026-07-02T05:50:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 118.347, "last_started_at": "2026-07-02T05:48:20+09:00", "pid": 41560, "process": {"exists": false, "pid": 41560}, "restart_count": 268, "restart_count_per_hour": 30, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 269}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 325.414, "sample_count": 994, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 364807, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:50:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T05:50:18+09:00", "last_frame_read_age_sec": 0.265, "last_frame_read_at": "2026-07-02T05:50:18+09:00", "mjpeg": {"age_sec": 0.256, "exists": true, "mtime": "2026-07-02T05:50:18.389325+09:00", "mtime_epoch": 1782939018.3893254, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182122}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:50:18+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1045.457, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 110, "pushed_frames": 110, "read_frames": 109, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.671, "pushed": 3.671, "read": 3.638}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.896, "count": 109, "max_ms": 12.996, "p95_ms": 12.098}, "gstreamer_push": {"avg_ms": 0.895, "count": 110, "max_ms": 2.868, "p95_ms": 0.908}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.543, "count": 109, "max_ms": 69.607, "p95_ms": 63.132}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.321, "count": 109, "max_ms": 96.668, "p95_ms": 94.665}, "yolo11_640_fire_11class": {"avg_ms": 53.387, "count": 109, "max_ms": 72.912, "p95_ms": 71.355}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.458, "count": 109, "max_ms": 99.156, "p95_ms": 94.166}}, "truncated_count": 0}, "processing_total": {"avg_ms": 266.352, "count": 110, "max_ms": 374.646, "p95_ms": 296.263}, "read_interval": {"avg_ms": 274.845, "count": 109, "max_ms": 383.251, "p95_ms": 305.322}, "triton_wait": {"avg_ms": 75.045, "count": 109, "max_ms": 99.169, "p95_ms": 95.148}, "visualize": {"avg_ms": 12.081, "count": 109, "max_ms": 14.855, "p95_ms": 12.933}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.965, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3040}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4513.547, "limit_mb": 8796093022207.996}, "detail": {"pid": 41667, "restart_count": 269, "start_count": 270}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 241, "h264_decode_error_count": 424, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:50:18+09:00", "last_error_summary": "[h264 @ 0x5650e45a5bc0] error while decoding MB 45 14, bytestream -5", "last_exit_at": "2026-07-02T05:50:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T05:50:21+09:00", "pid": 41667, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.492, "name": "ffmpeg", "pid": 41667, "rss_mb": 139.492, "state": "S (sleeping)", "threads": 106}, "restart_count": 269, "restart_count_per_hour": 31, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 270}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 325.414, "sample_count": 978, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 364807, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:50:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.751, "last_frame_pushed_at": "2026-07-02T05:50:18+09:00", "last_frame_read_age_sec": 5.009, "last_frame_read_at": "2026-07-02T05:50:18+09:00", "mjpeg": {"age_sec": 4.737, "exists": true, "mtime": "2026-07-02T05:50:18.653334+09:00", "mtime_epoch": 1782939018.6533341, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182259}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:50:23+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1045.457, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 93, "pushed_frames": 93, "read_frames": 92, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.102, "pushed": 3.102, "read": 3.069}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 11.056, "count": 92, "max_ms": 12.996, "p95_ms": 12.136}, "gstreamer_push": {"avg_ms": 0.905, "count": 93, "max_ms": 2.868, "p95_ms": 0.941}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.855, "count": 92, "max_ms": 66.782, "p95_ms": 61.334}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.345, "count": 92, "max_ms": 96.668, "p95_ms": 94.692}, "yolo11_640_fire_11class": {"avg_ms": 52.335, "count": 92, "max_ms": 72.079, "p95_ms": 70.657}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.842, "count": 92, "max_ms": 96.26, "p95_ms": 94.128}}, "truncated_count": 0}, "processing_total": {"avg_ms": 266.094, "count": 93, "max_ms": 374.646, "p95_ms": 296.263}, "read_interval": {"avg_ms": 274.408, "count": 92, "max_ms": 383.251, "p95_ms": 305.322}, "triton_wait": {"avg_ms": 73.684, "count": 92, "max_ms": 96.68, "p95_ms": 95.021}, "visualize": {"avg_ms": 12.113, "count": 92, "max_ms": 14.855, "p95_ms": 12.939}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.979, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3043}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4411.168, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55e024230100] error while decoding MB 20 39, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 242, "h264_decode_error_count": 425, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:52:22+09:00", "last_error_summary": "[h264 @ 0x55e024230100] error while decoding MB 20 39, bytestream -7", "last_exit_at": "2026-07-02T05:50:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.309, "last_started_at": "2026-07-02T05:50:21+09:00", "pid": 41667, "process": {"exists": false, "pid": 41667}, "restart_count": 269, "restart_count_per_hour": 30, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 270}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 318.364, "sample_count": 1006, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 365216, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:52:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-02T05:52:22+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-02T05:52:22+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-02T05:52:22.657251+09:00", "mtime_epoch": 1782939142.6572511, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183281}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:52:22+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1047.965, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 101, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.389, "pushed": 3.389, "read": 3.389}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 12.335, "count": 101, "max_ms": 14.194, "p95_ms": 13.837}, "gstreamer_push": {"avg_ms": 0.905, "count": 101, "max_ms": 3.149, "p95_ms": 0.93}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.591, "count": 101, "max_ms": 69.511, "p95_ms": 62.678}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.79, "count": 101, "max_ms": 101.134, "p95_ms": 96.677}, "yolo11_640_fire_11class": {"avg_ms": 54.878, "count": 101, "max_ms": 75.884, "p95_ms": 72.168}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.681, "count": 101, "max_ms": 98.913, "p95_ms": 95.402}}, "truncated_count": 0}, "processing_total": {"avg_ms": 287.889, "count": 101, "max_ms": 358.728, "p95_ms": 320.184}, "read_interval": {"avg_ms": 297.521, "count": 101, "max_ms": 370.466, "p95_ms": 329.181}, "triton_wait": {"avg_ms": 77.46, "count": 101, "max_ms": 101.144, "p95_ms": 96.688}, "visualize": {"avg_ms": 12.953, "count": 101, "max_ms": 14.891, "p95_ms": 14.383}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.806, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3043}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4410.461, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T05:52:22+09:00", "last_error_summary": "[h264 @ 0x55e024230100] error while decoding MB 20 39, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 242, "h264_decode_error_count": 425, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:52:22+09:00", "last_error_summary": "[h264 @ 0x55e024230100] error while decoding MB 20 39, bytestream -7", "last_exit_at": "2026-07-02T05:52:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.53, "last_started_at": "2026-07-02T05:50:21+09:00", "pid": 41667, "process": {"exists": false, "pid": 41667}, "restart_count": 269, "restart_count_per_hour": 30, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 270}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 318.364, "sample_count": 1006, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 365217, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:52:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T05:52:22+09:00", "last_frame_read_age_sec": 0.272, "last_frame_read_at": "2026-07-02T05:52:22+09:00", "mjpeg": {"age_sec": 0.262, "exists": true, "mtime": "2026-07-02T05:52:22.657251+09:00", "mtime_epoch": 1782939142.6572511, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183281}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:52:22+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1047.965, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 100, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.397, "pushed": 3.397, "read": 3.364}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 12.315, "count": 101, "max_ms": 14.194, "p95_ms": 13.837}, "gstreamer_push": {"avg_ms": 0.905, "count": 101, "max_ms": 3.149, "p95_ms": 0.93}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.407, "count": 101, "max_ms": 69.511, "p95_ms": 62.678}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.569, "count": 101, "max_ms": 101.134, "p95_ms": 96.677}, "yolo11_640_fire_11class": {"avg_ms": 54.668, "count": 101, "max_ms": 75.884, "p95_ms": 72.168}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.465, "count": 101, "max_ms": 98.913, "p95_ms": 95.402}}, "truncated_count": 0}, "processing_total": {"avg_ms": 287.621, "count": 101, "max_ms": 358.728, "p95_ms": 320.184}, "read_interval": {"avg_ms": 297.441, "count": 100, "max_ms": 370.466, "p95_ms": 329.181}, "triton_wait": {"avg_ms": 77.239, "count": 101, "max_ms": 101.144, "p95_ms": 96.688}, "visualize": {"avg_ms": 12.952, "count": 101, "max_ms": 14.891, "p95_ms": 14.383}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.729, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3043}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4410.453, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 242, "h264_decode_error_count": 425, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:52:22+09:00", "last_error_summary": "[h264 @ 0x55e024230100] error while decoding MB 20 39, bytestream -7", "last_exit_at": "2026-07-02T05:52:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.533, "last_started_at": "2026-07-02T05:50:21+09:00", "pid": 41667, "process": {"exists": false, "pid": 41667}, "restart_count": 269, "restart_count_per_hour": 30, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 270}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 318.364, "sample_count": 1006, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 365217, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:52:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T05:52:22+09:00", "last_frame_read_age_sec": 0.275, "last_frame_read_at": "2026-07-02T05:52:22+09:00", "mjpeg": {"age_sec": 0.265, "exists": true, "mtime": "2026-07-02T05:52:22.657251+09:00", "mtime_epoch": 1782939142.6572511, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183281}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:52:22+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1047.965, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 101, "pushed_frames": 101, "read_frames": 100, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.397, "pushed": 3.397, "read": 3.363}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 12.315, "count": 101, "max_ms": 14.194, "p95_ms": 13.837}, "gstreamer_push": {"avg_ms": 0.905, "count": 101, "max_ms": 3.149, "p95_ms": 0.93}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.407, "count": 101, "max_ms": 69.511, "p95_ms": 62.678}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.569, "count": 101, "max_ms": 101.134, "p95_ms": 96.677}, "yolo11_640_fire_11class": {"avg_ms": 54.668, "count": 101, "max_ms": 75.884, "p95_ms": 72.168}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.465, "count": 101, "max_ms": 98.913, "p95_ms": 95.402}}, "truncated_count": 0}, "processing_total": {"avg_ms": 287.621, "count": 101, "max_ms": 358.728, "p95_ms": 320.184}, "read_interval": {"avg_ms": 297.441, "count": 100, "max_ms": 370.466, "p95_ms": 329.181}, "triton_wait": {"avg_ms": 77.239, "count": 101, "max_ms": 101.144, "p95_ms": 96.688}, "visualize": {"avg_ms": 12.952, "count": 101, "max_ms": 14.891, "p95_ms": 14.383}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.732, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 3043}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4515.598, "limit_mb": 8796093022207.996}, "detail": {"pid": 41774, "restart_count": 270, "start_count": 271}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 242, "h264_decode_error_count": 425, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:52:22+09:00", "last_error_summary": "[h264 @ 0x55e024230100] error while decoding MB 20 39, bytestream -7", "last_exit_at": "2026-07-02T05:52:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T05:52:25+09:00", "pid": 41774, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.5, "name": "ffmpeg", "pid": 41774, "rss_mb": 136.5, "state": "S (sleeping)", "threads": 106}, "restart_count": 270, "restart_count_per_hour": 31, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 271}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 318.875, "sample_count": 989, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 365217, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:52:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.85, "last_frame_pushed_at": "2026-07-02T05:52:22+09:00", "last_frame_read_age_sec": 5.119, "last_frame_read_at": "2026-07-02T05:52:22+09:00", "mjpeg": {"age_sec": 4.833, "exists": true, "mtime": "2026-07-02T05:52:22.933259+09:00", "mtime_epoch": 1782939142.9332595, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 183352}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:52:27+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1047.965, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 85, "pushed_frames": 85, "read_frames": 84, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.85, "pushed": 2.85, "read": 2.816}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 12.25, "count": 84, "max_ms": 14.194, "p95_ms": 13.837}, "gstreamer_push": {"avg_ms": 0.912, "count": 85, "max_ms": 3.149, "p95_ms": 0.939}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.605, "count": 84, "max_ms": 69.511, "p95_ms": 62.678}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.33, "count": 84, "max_ms": 101.134, "p95_ms": 99.141}, "yolo11_640_fire_11class": {"avg_ms": 55.101, "count": 84, "max_ms": 75.884, "p95_ms": 72.44}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.715, "count": 84, "max_ms": 98.913, "p95_ms": 95.406}}, "truncated_count": 0}, "processing_total": {"avg_ms": 288.064, "count": 85, "max_ms": 358.728, "p95_ms": 320.41}, "read_interval": {"avg_ms": 297.925, "count": 84, "max_ms": 370.466, "p95_ms": 329.389}, "triton_wait": {"avg_ms": 77.708, "count": 84, "max_ms": 101.144, "p95_ms": 99.152}, "visualize": {"avg_ms": 12.879, "count": 84, "max_ms": 14.891, "p95_ms": 14.399}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.829, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 3046}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4559.895, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x555d7b0b5e40] cabac decode of qscale diff failed at 49 33 [h264 @ 0x555d7b0b5e40] error while decoding MB 49 33, bytestream 2055", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 243, "h264_decode_error_count": 427, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:54:33+09:00", "last_error_summary": "[h264 @ 0x555d7b0b5e40] cabac decode of qscale diff failed at 49 33 [h264 @ 0x555d7b0b5e40] error while decoding MB 49 33, bytestream 2055", "last_exit_at": "2026-07-02T05:52:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 127.934, "last_started_at": "2026-07-02T05:52:25+09:00", "pid": 41774, "process": {"exists": false, "pid": 41774}, "restart_count": 270, "restart_count_per_hour": 30, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 271}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 375.086, "sample_count": 953, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 365597, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:54:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.068, "last_frame_pushed_at": "2026-07-02T05:54:33+09:00", "last_frame_read_age_sec": 0.06, "last_frame_read_at": "2026-07-02T05:54:33+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-02T05:54:33.653106+09:00", "mtime_epoch": 1782939273.6531065, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186031}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:54:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1192.5, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 93, "pushed_frames": 93, "read_frames": 93, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.115, "pushed": 3.115, "read": 3.115}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.006, "count": 92, "max_ms": 23.561, "p95_ms": 17.76}, "gstreamer_push": {"avg_ms": 0.877, "count": 93, "max_ms": 1.234, "p95_ms": 0.945}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.709, "count": 92, "max_ms": 69.265, "p95_ms": 62.162}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.835, "count": 92, "max_ms": 101.994, "p95_ms": 100.694}, "yolo11_640_fire_11class": {"avg_ms": 52.97, "count": 92, "max_ms": 73.494, "p95_ms": 70.938}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.243, "count": 92, "max_ms": 98.697, "p95_ms": 94.117}}, "truncated_count": 0}, "processing_total": {"avg_ms": 314.811, "count": 93, "max_ms": 371.974, "p95_ms": 361.804}, "read_interval": {"avg_ms": 323.911, "count": 93, "max_ms": 380.835, "p95_ms": 370.693}, "triton_wait": {"avg_ms": 78.06, "count": 92, "max_ms": 102.006, "p95_ms": 100.706}, "visualize": {"avg_ms": 14.118, "count": 92, "max_ms": 19.017, "p95_ms": 16.242}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.859, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 3046}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4558.504, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T05:54:33+09:00", "last_error_summary": "[h264 @ 0x555d7b0b5e40] cabac decode of qscale diff failed at 49 33 [h264 @ 0x555d7b0b5e40] error while decoding MB 49 33, bytestream 2055", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 243, "h264_decode_error_count": 427, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:54:33+09:00", "last_error_summary": "[h264 @ 0x555d7b0b5e40] cabac decode of qscale diff failed at 49 33 [h264 @ 0x555d7b0b5e40] error while decoding MB 49 33, bytestream 2055", "last_exit_at": "2026-07-02T05:54:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.159, "last_started_at": "2026-07-02T05:52:25+09:00", "pid": 41774, "process": {"exists": false, "pid": 41774}, "restart_count": 270, "restart_count_per_hour": 30, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 271}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 375.086, "sample_count": 953, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 365598, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:54:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T05:54:33+09:00", "last_frame_read_age_sec": 0.285, "last_frame_read_at": "2026-07-02T05:54:33+09:00", "mjpeg": {"age_sec": 0.271, "exists": true, "mtime": "2026-07-02T05:54:33.653106+09:00", "mtime_epoch": 1782939273.6531065, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186031}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:54:33+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1192.5, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 93, "pushed_frames": 93, "read_frames": 92, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.126, "pushed": 3.126, "read": 3.093}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 13.988, "count": 93, "max_ms": 23.561, "p95_ms": 17.76}, "gstreamer_push": {"avg_ms": 0.877, "count": 93, "max_ms": 1.234, "p95_ms": 0.945}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.63, "count": 93, "max_ms": 69.265, "p95_ms": 62.162}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.748, "count": 93, "max_ms": 101.994, "p95_ms": 100.694}, "yolo11_640_fire_11class": {"avg_ms": 52.895, "count": 93, "max_ms": 73.494, "p95_ms": 70.938}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.18, "count": 93, "max_ms": 98.697, "p95_ms": 94.117}}, "truncated_count": 0}, "processing_total": {"avg_ms": 314.319, "count": 93, "max_ms": 371.974, "p95_ms": 361.804}, "read_interval": {"avg_ms": 323.782, "count": 92, "max_ms": 380.835, "p95_ms": 370.693}, "triton_wait": {"avg_ms": 77.971, "count": 93, "max_ms": 102.006, "p95_ms": 100.706}, "visualize": {"avg_ms": 14.098, "count": 93, "max_ms": 19.017, "p95_ms": 16.242}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.747, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 3046}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4558.496, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 243, "h264_decode_error_count": 427, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:54:33+09:00", "last_error_summary": "[h264 @ 0x555d7b0b5e40] cabac decode of qscale diff failed at 49 33 [h264 @ 0x555d7b0b5e40] error while decoding MB 49 33, bytestream 2055", "last_exit_at": "2026-07-02T05:54:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.162, "last_started_at": "2026-07-02T05:52:25+09:00", "pid": 41774, "process": {"exists": false, "pid": 41774}, "restart_count": 270, "restart_count_per_hour": 30, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 271}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 375.086, "sample_count": 953, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 365598, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:54:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T05:54:33+09:00", "last_frame_read_age_sec": 0.288, "last_frame_read_at": "2026-07-02T05:54:33+09:00", "mjpeg": {"age_sec": 0.275, "exists": true, "mtime": "2026-07-02T05:54:33.653106+09:00", "mtime_epoch": 1782939273.6531065, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186031}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:54:33+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1192.5, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 93, "pushed_frames": 93, "read_frames": 92, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.126, "pushed": 3.126, "read": 3.092}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 13.988, "count": 93, "max_ms": 23.561, "p95_ms": 17.76}, "gstreamer_push": {"avg_ms": 0.877, "count": 93, "max_ms": 1.234, "p95_ms": 0.945}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.63, "count": 93, "max_ms": 69.265, "p95_ms": 62.162}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.748, "count": 93, "max_ms": 101.994, "p95_ms": 100.694}, "yolo11_640_fire_11class": {"avg_ms": 52.895, "count": 93, "max_ms": 73.494, "p95_ms": 70.938}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.18, "count": 93, "max_ms": 98.697, "p95_ms": 94.117}}, "truncated_count": 0}, "processing_total": {"avg_ms": 314.319, "count": 93, "max_ms": 371.974, "p95_ms": 361.804}, "read_interval": {"avg_ms": 323.782, "count": 92, "max_ms": 380.835, "p95_ms": 370.693}, "triton_wait": {"avg_ms": 77.971, "count": 93, "max_ms": 102.006, "p95_ms": 100.706}, "visualize": {"avg_ms": 14.098, "count": 93, "max_ms": 19.017, "p95_ms": 16.242}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.75, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 4, "tracking_summary": 3046}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4669.758, "limit_mb": 8796093022207.996}, "detail": {"pid": 41941, "restart_count": 271, "start_count": 272}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 243, "h264_decode_error_count": 427, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:54:33+09:00", "last_error_summary": "[h264 @ 0x555d7b0b5e40] cabac decode of qscale diff failed at 49 33 [h264 @ 0x555d7b0b5e40] error while decoding MB 49 33, bytestream 2055", "last_exit_at": "2026-07-02T05:54:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T05:54:35+09:00", "pid": 41941, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.82, "name": "ffmpeg", "pid": 41941, "rss_mb": 138.82, "state": "S (sleeping)", "threads": 106}, "restart_count": 271, "restart_count_per_hour": 31, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 272}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 375.395, "sample_count": 938, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 365598, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:54:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.067, "last_frame_pushed_at": "2026-07-02T05:54:33+09:00", "last_frame_read_age_sec": 4.349, "last_frame_read_at": "2026-07-02T05:54:33+09:00", "mjpeg": {"age_sec": 4.051, "exists": true, "mtime": "2026-07-02T05:54:33.937114+09:00", "mtime_epoch": 1782939273.9371145, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186104}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:54:37+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1192.5, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 82, "pushed_frames": 82, "read_frames": 81, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.734, "pushed": 2.734, "read": 2.701}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 13.574, "count": 81, "max_ms": 18.581, "p95_ms": 15.158}, "gstreamer_push": {"avg_ms": 0.872, "count": 82, "max_ms": 1.233, "p95_ms": 0.93}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.324, "count": 81, "max_ms": 68.803, "p95_ms": 59.729}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.135, "count": 81, "max_ms": 101.651, "p95_ms": 98.902}, "yolo11_640_fire_11class": {"avg_ms": 52.457, "count": 81, "max_ms": 73.494, "p95_ms": 68.921}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.585, "count": 81, "max_ms": 98.697, "p95_ms": 94.117}}, "truncated_count": 0}, "processing_total": {"avg_ms": 311.289, "count": 82, "max_ms": 369.055, "p95_ms": 355.793}, "read_interval": {"avg_ms": 320.681, "count": 81, "max_ms": 377.745, "p95_ms": 364.76}, "triton_wait": {"avg_ms": 78.181, "count": 81, "max_ms": 101.663, "p95_ms": 98.918}, "visualize": {"avg_ms": 14.084, "count": 81, "max_ms": 19.017, "p95_ms": 16.142}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 3049}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4650.918, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5577f1587540] error while decoding MB 27 25, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 244, "h264_decode_error_count": 428, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:56:40+09:00", "last_error_summary": "[h264 @ 0x5577f1587540] error while decoding MB 27 25, bytestream -7", "last_exit_at": "2026-07-02T05:54:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.025, "last_started_at": "2026-07-02T05:54:35+09:00", "pid": 41941, "process": {"exists": false, "pid": 41941}, "restart_count": 271, "restart_count_per_hour": 30, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 272}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 443.075, "sample_count": 843, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 365915, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:56:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-02T05:56:39+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-02T05:56:39+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-02T05:56:39.968609+09:00", "mtime_epoch": 1782939399.9686093, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187328}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:56:40+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1281.688, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 88, "pushed_frames": 88, "read_frames": 88, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.967, "pushed": 2.967, "read": 2.967}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 15.38, "count": 88, "max_ms": 22.064, "p95_ms": 17.872}, "gstreamer_push": {"avg_ms": 0.87, "count": 88, "max_ms": 0.953, "p95_ms": 0.915}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.695, "count": 88, "max_ms": 69.025, "p95_ms": 59.686}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.071, "count": 88, "max_ms": 101.436, "p95_ms": 94.809}, "yolo11_640_fire_11class": {"avg_ms": 52.247, "count": 88, "max_ms": 72.662, "p95_ms": 69.506}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.395, "count": 88, "max_ms": 99.683, "p95_ms": 92.566}}, "truncated_count": 0}, "processing_total": {"avg_ms": 331.112, "count": 88, "max_ms": 381.932, "p95_ms": 366.335}, "read_interval": {"avg_ms": 340.667, "count": 88, "max_ms": 390.761, "p95_ms": 376.546}, "triton_wait": {"avg_ms": 77.137, "count": 88, "max_ms": 101.448, "p95_ms": 94.821}, "visualize": {"avg_ms": 14.432, "count": 88, "max_ms": 19.015, "p95_ms": 15.951}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.663, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 3049}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4652.66, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T05:56:40+09:00", "last_error_summary": "[h264 @ 0x5577f1587540] error while decoding MB 27 25, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 244, "h264_decode_error_count": 428, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:56:40+09:00", "last_error_summary": "[h264 @ 0x5577f1587540] error while decoding MB 27 25, bytestream -7", "last_exit_at": "2026-07-02T05:56:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.311, "last_started_at": "2026-07-02T05:54:35+09:00", "pid": 41941, "process": {"exists": false, "pid": 41941}, "restart_count": 271, "restart_count_per_hour": 30, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 272}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 443.075, "sample_count": 843, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 365916, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:56:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T05:56:40+09:00", "last_frame_read_age_sec": 0.336, "last_frame_read_at": "2026-07-02T05:56:39+09:00", "mjpeg": {"age_sec": 0.329, "exists": true, "mtime": "2026-07-02T05:56:39.968609+09:00", "mtime_epoch": 1782939399.9686093, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187328}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:56:40+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1281.688, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 89, "pushed_frames": 89, "read_frames": 88, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.972, "pushed": 2.972, "read": 2.938}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 15.342, "count": 88, "max_ms": 22.064, "p95_ms": 17.774}, "gstreamer_push": {"avg_ms": 0.87, "count": 89, "max_ms": 0.953, "p95_ms": 0.915}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.875, "count": 88, "max_ms": 69.025, "p95_ms": 59.686}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.184, "count": 88, "max_ms": 101.436, "p95_ms": 94.809}, "yolo11_640_fire_11class": {"avg_ms": 52.428, "count": 88, "max_ms": 72.662, "p95_ms": 69.506}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.575, "count": 88, "max_ms": 99.683, "p95_ms": 92.566}}, "truncated_count": 0}, "processing_total": {"avg_ms": 331.107, "count": 89, "max_ms": 381.932, "p95_ms": 366.335}, "read_interval": {"avg_ms": 340.667, "count": 88, "max_ms": 390.761, "p95_ms": 376.546}, "triton_wait": {"avg_ms": 77.25, "count": 88, "max_ms": 101.448, "p95_ms": 94.821}, "visualize": {"avg_ms": 14.423, "count": 88, "max_ms": 19.015, "p95_ms": 15.951}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.949, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 3049}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4652.777, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 244, "h264_decode_error_count": 428, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:56:40+09:00", "last_error_summary": "[h264 @ 0x5577f1587540] error while decoding MB 27 25, bytestream -7", "last_exit_at": "2026-07-02T05:56:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.314, "last_started_at": "2026-07-02T05:54:35+09:00", "pid": 41941, "process": {"exists": false, "pid": 41941}, "restart_count": 271, "restart_count_per_hour": 30, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 272}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 443.075, "sample_count": 843, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 365916, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:56:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-02T05:56:40+09:00", "last_frame_read_age_sec": 0.34, "last_frame_read_at": "2026-07-02T05:56:39+09:00", "mjpeg": {"age_sec": 0.332, "exists": true, "mtime": "2026-07-02T05:56:39.968609+09:00", "mtime_epoch": 1782939399.9686093, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187328}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:56:40+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1281.688, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 89, "pushed_frames": 89, "read_frames": 88, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.971, "pushed": 2.971, "read": 2.938}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 15.342, "count": 88, "max_ms": 22.064, "p95_ms": 17.774}, "gstreamer_push": {"avg_ms": 0.87, "count": 89, "max_ms": 0.953, "p95_ms": 0.915}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.875, "count": 88, "max_ms": 69.025, "p95_ms": 59.686}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.184, "count": 88, "max_ms": 101.436, "p95_ms": 94.809}, "yolo11_640_fire_11class": {"avg_ms": 52.428, "count": 88, "max_ms": 72.662, "p95_ms": 69.506}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.575, "count": 88, "max_ms": 99.683, "p95_ms": 92.566}}, "truncated_count": 0}, "processing_total": {"avg_ms": 331.107, "count": 89, "max_ms": 381.932, "p95_ms": 366.335}, "read_interval": {"avg_ms": 340.667, "count": 88, "max_ms": 390.761, "p95_ms": 376.546}, "triton_wait": {"avg_ms": 77.25, "count": 88, "max_ms": 101.448, "p95_ms": 94.821}, "visualize": {"avg_ms": 14.423, "count": 88, "max_ms": 19.015, "p95_ms": 15.951}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.952, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 3049}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4766.719, "limit_mb": 8796093022207.996}, "detail": {"pid": 42108, "restart_count": 272, "start_count": 273}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 244, "h264_decode_error_count": 428, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:56:40+09:00", "last_error_summary": "[h264 @ 0x5577f1587540] error while decoding MB 27 25, bytestream -7", "last_exit_at": "2026-07-02T05:56:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T05:56:42+09:00", "pid": 42108, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.32, "name": "ffmpeg", "pid": 42108, "rss_mb": 139.32, "state": "S (sleeping)", "threads": 106}, "restart_count": 272, "restart_count_per_hour": 31, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 273}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 443.292, "sample_count": 827, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 365916, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:56:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.635, "last_frame_pushed_at": "2026-07-02T05:56:40+09:00", "last_frame_read_age_sec": 4.967, "last_frame_read_at": "2026-07-02T05:56:39+09:00", "mjpeg": {"age_sec": 4.616, "exists": true, "mtime": "2026-07-02T05:56:40.312619+09:00", "mtime_epoch": 1782939400.3126187, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184831}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:56:44+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1281.688, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 75, "pushed_frames": 75, "read_frames": 74, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.521, "pushed": 2.521, "read": 2.487}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 15.049, "count": 75, "max_ms": 17.888, "p95_ms": 16.444}, "gstreamer_push": {"avg_ms": 0.87, "count": 75, "max_ms": 0.953, "p95_ms": 0.915}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.216, "count": 75, "max_ms": 69.025, "p95_ms": 62.07}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.855, "count": 75, "max_ms": 101.436, "p95_ms": 95.539}, "yolo11_640_fire_11class": {"avg_ms": 52.914, "count": 75, "max_ms": 72.662, "p95_ms": 69.636}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.308, "count": 75, "max_ms": 99.683, "p95_ms": 92.712}}, "truncated_count": 0}, "processing_total": {"avg_ms": 329.875, "count": 75, "max_ms": 381.932, "p95_ms": 363.415}, "read_interval": {"avg_ms": 339.411, "count": 74, "max_ms": 390.761, "p95_ms": 373.447}, "triton_wait": {"avg_ms": 77.929, "count": 75, "max_ms": 101.448, "p95_ms": 95.551}, "visualize": {"avg_ms": 14.454, "count": 75, "max_ms": 19.015, "p95_ms": 15.957}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.754, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 3052}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4643.855, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5639baa15700] error while decoding MB 57 29, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 245, "h264_decode_error_count": 429, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:58:41+09:00", "last_error_summary": "[h264 @ 0x5639baa15700] error while decoding MB 57 29, bytestream -5", "last_exit_at": "2026-07-02T05:56:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 118.164, "last_started_at": "2026-07-02T05:56:42+09:00", "pid": 42108, "process": {"exists": false, "pid": 42108}, "restart_count": 272, "restart_count_per_hour": 30, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 273}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 428.487, "sample_count": 884, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 366326, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:58:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.072, "last_frame_pushed_at": "2026-07-02T05:58:41+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-02T05:58:41+09:00", "mjpeg": {"age_sec": 0.052, "exists": true, "mtime": "2026-07-02T05:58:41.039812+09:00", "mtime_epoch": 1782939521.0398123, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188333}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:58:41+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1268.633, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 99, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.325, "pushed": 3.325, "read": 3.325}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 12.614, "count": 99, "max_ms": 14.566, "p95_ms": 13.947}, "gstreamer_push": {"avg_ms": 0.88, "count": 99, "max_ms": 1.634, "p95_ms": 0.918}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.939, "count": 99, "max_ms": 70.194, "p95_ms": 65.295}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.398, "count": 99, "max_ms": 104.19, "p95_ms": 100.245}, "yolo11_640_fire_11class": {"avg_ms": 56.241, "count": 99, "max_ms": 78.188, "p95_ms": 72.259}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.054, "count": 99, "max_ms": 101.728, "p95_ms": 98.913}}, "truncated_count": 0}, "processing_total": {"avg_ms": 293.415, "count": 99, "max_ms": 324.066, "p95_ms": 318.62}, "read_interval": {"avg_ms": 302.725, "count": 99, "max_ms": 333.383, "p95_ms": 328.331}, "triton_wait": {"avg_ms": 81.601, "count": 99, "max_ms": 104.201, "p95_ms": 100.347}, "visualize": {"avg_ms": 13.125, "count": 99, "max_ms": 20.463, "p95_ms": 14.24}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.771, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 3052}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4642.551, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T05:58:41+09:00", "last_error_summary": "[h264 @ 0x5639baa15700] error while decoding MB 57 29, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 245, "h264_decode_error_count": 429, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:58:41+09:00", "last_error_summary": "[h264 @ 0x5639baa15700] error while decoding MB 57 29, bytestream -5", "last_exit_at": "2026-07-02T05:58:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 118.393, "last_started_at": "2026-07-02T05:56:42+09:00", "pid": 42108, "process": {"exists": false, "pid": 42108}, "restart_count": 272, "restart_count_per_hour": 30, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 273}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 428.487, "sample_count": 885, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 366327, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:58:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T05:58:41+09:00", "last_frame_read_age_sec": 0.293, "last_frame_read_at": "2026-07-02T05:58:41+09:00", "mjpeg": {"age_sec": 0.28, "exists": true, "mtime": "2026-07-02T05:58:41.039812+09:00", "mtime_epoch": 1782939521.0398123, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188333}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:58:41+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1268.633, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.333, "pushed": 3.333, "read": 3.3}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 12.628, "count": 99, "max_ms": 14.566, "p95_ms": 13.947}, "gstreamer_push": {"avg_ms": 0.881, "count": 100, "max_ms": 1.634, "p95_ms": 0.918}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.915, "count": 99, "max_ms": 70.194, "p95_ms": 65.295}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.397, "count": 99, "max_ms": 104.19, "p95_ms": 100.245}, "yolo11_640_fire_11class": {"avg_ms": 56.202, "count": 99, "max_ms": 78.188, "p95_ms": 72.259}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.064, "count": 99, "max_ms": 101.728, "p95_ms": 98.913}}, "truncated_count": 0}, "processing_total": {"avg_ms": 293.359, "count": 100, "max_ms": 324.066, "p95_ms": 318.001}, "read_interval": {"avg_ms": 302.725, "count": 99, "max_ms": 333.383, "p95_ms": 328.331}, "triton_wait": {"avg_ms": 81.6, "count": 99, "max_ms": 104.201, "p95_ms": 100.347}, "visualize": {"avg_ms": 13.13, "count": 99, "max_ms": 20.463, "p95_ms": 14.24}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 3052}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4642.559, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 245, "h264_decode_error_count": 429, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:58:41+09:00", "last_error_summary": "[h264 @ 0x5639baa15700] error while decoding MB 57 29, bytestream -5", "last_exit_at": "2026-07-02T05:58:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 118.398, "last_started_at": "2026-07-02T05:56:42+09:00", "pid": 42108, "process": {"exists": false, "pid": 42108}, "restart_count": 272, "restart_count_per_hour": 30, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 273}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 428.487, "sample_count": 885, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 366327, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:58:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-02T05:58:41+09:00", "last_frame_read_age_sec": 0.298, "last_frame_read_at": "2026-07-02T05:58:41+09:00", "mjpeg": {"age_sec": 0.286, "exists": true, "mtime": "2026-07-02T05:58:41.039812+09:00", "mtime_epoch": 1782939521.0398123, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188333}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:58:41+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1268.633, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 99, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.301, "pushed": 3.301, "read": 3.301}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 12.628, "count": 99, "max_ms": 14.566, "p95_ms": 13.947}, "gstreamer_push": {"avg_ms": 0.881, "count": 99, "max_ms": 1.634, "p95_ms": 0.918}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.915, "count": 99, "max_ms": 70.194, "p95_ms": 65.295}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.397, "count": 99, "max_ms": 104.19, "p95_ms": 100.245}, "yolo11_640_fire_11class": {"avg_ms": 56.202, "count": 99, "max_ms": 78.188, "p95_ms": 72.259}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.064, "count": 99, "max_ms": 101.728, "p95_ms": 98.913}}, "truncated_count": 0}, "processing_total": {"avg_ms": 293.665, "count": 99, "max_ms": 324.066, "p95_ms": 318.62}, "read_interval": {"avg_ms": 302.725, "count": 99, "max_ms": 333.383, "p95_ms": 328.331}, "triton_wait": {"avg_ms": 81.6, "count": 99, "max_ms": 104.201, "p95_ms": 100.347}, "visualize": {"avg_ms": 13.13, "count": 99, "max_ms": 20.463, "p95_ms": 14.24}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 3052}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4652.691, "limit_mb": 8796093022207.996}, "detail": {"pid": 42215, "restart_count": 273, "start_count": 274}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 245, "h264_decode_error_count": 429, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T05:58:41+09:00", "last_error_summary": "[h264 @ 0x5639baa15700] error while decoding MB 57 29, bytestream -5", "last_exit_at": "2026-07-02T05:58:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T05:58:43+09:00", "pid": 42215, "process": {"exists": true, "fd_count": 4, "hwm_mb": 43.664, "name": "ffmpeg", "pid": 42215, "rss_mb": 43.664, "state": "S (sleeping)", "threads": 1}, "restart_count": 273, "restart_count_per_hour": 31, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 274}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 428.487, "sample_count": 873, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 366327, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T05:58:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.257, "last_frame_pushed_at": "2026-07-02T05:58:41+09:00", "last_frame_read_age_sec": 4.546, "last_frame_read_at": "2026-07-02T05:58:41+09:00", "mjpeg": {"age_sec": 4.238, "exists": true, "mtime": "2026-07-02T05:58:41.335820+09:00", "mtime_epoch": 1782939521.33582, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188028}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T05:58:45+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1496.441, "name": "celery", "pid": 237, "rss_mb": 1268.633, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 85, "pushed_frames": 85, "read_frames": 84, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.861, "pushed": 2.861, "read": 2.827}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 12.542, "count": 85, "max_ms": 14.366, "p95_ms": 13.637}, "gstreamer_push": {"avg_ms": 0.881, "count": 85, "max_ms": 1.634, "p95_ms": 0.912}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.718, "count": 85, "max_ms": 69.089, "p95_ms": 63.669}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.144, "count": 85, "max_ms": 104.19, "p95_ms": 100.245}, "yolo11_640_fire_11class": {"avg_ms": 56.44, "count": 85, "max_ms": 78.188, "p95_ms": 71.507}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.57, "count": 85, "max_ms": 101.728, "p95_ms": 98.913}}, "truncated_count": 0}, "processing_total": {"avg_ms": 293.767, "count": 85, "max_ms": 324.066, "p95_ms": 317.897}, "read_interval": {"avg_ms": 303.119, "count": 84, "max_ms": 333.383, "p95_ms": 327.172}, "triton_wait": {"avg_ms": 82.321, "count": 85, "max_ms": 104.201, "p95_ms": 100.347}, "visualize": {"avg_ms": 13.1, "count": 85, "max_ms": 20.463, "p95_ms": 14.015}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.712, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 20, "tracking_summary": 3055}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5167.918, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55a22bdbbf80] error while decoding MB 79 41, bytestream -33", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 246, "h264_decode_error_count": 430, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:01:01+09:00", "last_error_summary": "[h264 @ 0x55a22bdbbf80] error while decoding MB 79 41, bytestream -33", "last_exit_at": "2026-07-02T05:58:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 137.819, "last_started_at": "2026-07-02T05:58:43+09:00", "pid": 42215, "process": {"exists": false, "pid": 42215}, "restart_count": 273, "restart_count_per_hour": 30, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 274}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 457.754, "sample_count": 835, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 366636, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:01:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-02T06:01:01+09:00", "last_frame_read_age_sec": 0.047, "last_frame_read_at": "2026-07-02T06:01:01+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-02T06:01:01.351377+09:00", "mtime_epoch": 1782939661.3513775, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189965}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:01:01+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1650.719, "name": "celery", "pid": 237, "rss_mb": 1595.664, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 71, "pushed_frames": 71, "read_frames": 71, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.388, "pushed": 2.388, "read": 2.388}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 21.505, "count": 70, "max_ms": 27.358, "p95_ms": 25.506}, "gstreamer_push": {"avg_ms": 0.877, "count": 71, "max_ms": 1.213, "p95_ms": 0.921}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.492, "count": 70, "max_ms": 70.606, "p95_ms": 66.139}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.483, "count": 70, "max_ms": 102.746, "p95_ms": 101.291}, "yolo11_640_fire_11class": {"avg_ms": 53.834, "count": 70, "max_ms": 69.346, "p95_ms": 68.106}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.051, "count": 70, "max_ms": 100.718, "p95_ms": 94.506}}, "truncated_count": 0}, "processing_total": {"avg_ms": 413.836, "count": 71, "max_ms": 502.714, "p95_ms": 473.253}, "read_interval": {"avg_ms": 423.473, "count": 71, "max_ms": 513.314, "p95_ms": 483.908}, "triton_wait": {"avg_ms": 79.244, "count": 70, "max_ms": 102.757, "p95_ms": 101.304}, "visualize": {"avg_ms": 20.459, "count": 70, "max_ms": 24.899, "p95_ms": 22.493}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.736, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 20, "tracking_summary": 3055}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5167.113, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T06:01:01+09:00", "last_error_summary": "[h264 @ 0x55a22bdbbf80] error while decoding MB 79 41, bytestream -33", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 246, "h264_decode_error_count": 430, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:01:01+09:00", "last_error_summary": "[h264 @ 0x55a22bdbbf80] error while decoding MB 79 41, bytestream -33", "last_exit_at": "2026-07-02T06:01:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 138.203, "last_started_at": "2026-07-02T05:58:43+09:00", "pid": 42215, "process": {"exists": false, "pid": 42215}, "restart_count": 273, "restart_count_per_hour": 30, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 274}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 457.754, "sample_count": 835, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 366637, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:01:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.002, "last_frame_pushed_at": "2026-07-02T06:01:01+09:00", "last_frame_read_age_sec": 0.432, "last_frame_read_at": "2026-07-02T06:01:01+09:00", "mjpeg": {"age_sec": 0.424, "exists": true, "mtime": "2026-07-02T06:01:01.351377+09:00", "mtime_epoch": 1782939661.3513775, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189965}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:01:01+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1650.719, "name": "celery", "pid": 237, "rss_mb": 1595.664, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 71, "pushed_frames": 71, "read_frames": 70, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.391, "pushed": 2.391, "read": 2.357}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 21.521, "count": 71, "max_ms": 27.358, "p95_ms": 25.506}, "gstreamer_push": {"avg_ms": 0.877, "count": 71, "max_ms": 1.213, "p95_ms": 0.921}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.459, "count": 71, "max_ms": 70.606, "p95_ms": 66.139}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.513, "count": 71, "max_ms": 102.746, "p95_ms": 101.291}, "yolo11_640_fire_11class": {"avg_ms": 53.777, "count": 71, "max_ms": 69.346, "p95_ms": 68.106}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.005, "count": 71, "max_ms": 100.718, "p95_ms": 94.506}}, "truncated_count": 0}, "processing_total": {"avg_ms": 414.535, "count": 71, "max_ms": 502.714, "p95_ms": 473.253}, "read_interval": {"avg_ms": 423.984, "count": 70, "max_ms": 513.314, "p95_ms": 483.908}, "triton_wait": {"avg_ms": 79.264, "count": 71, "max_ms": 102.757, "p95_ms": 101.304}, "visualize": {"avg_ms": 20.471, "count": 71, "max_ms": 24.899, "p95_ms": 22.493}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.697, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 20, "tracking_summary": 3055}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5167.105, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 246, "h264_decode_error_count": 430, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:01:01+09:00", "last_error_summary": "[h264 @ 0x55a22bdbbf80] error while decoding MB 79 41, bytestream -33", "last_exit_at": "2026-07-02T06:01:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 138.206, "last_started_at": "2026-07-02T05:58:43+09:00", "pid": 42215, "process": {"exists": false, "pid": 42215}, "restart_count": 273, "restart_count_per_hour": 30, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 274}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 457.754, "sample_count": 835, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 366637, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:01:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T06:01:01+09:00", "last_frame_read_age_sec": 0.435, "last_frame_read_at": "2026-07-02T06:01:01+09:00", "mjpeg": {"age_sec": 0.427, "exists": true, "mtime": "2026-07-02T06:01:01.351377+09:00", "mtime_epoch": 1782939661.3513775, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189965}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:01:01+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1650.719, "name": "celery", "pid": 237, "rss_mb": 1595.664, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 71, "pushed_frames": 71, "read_frames": 70, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.391, "pushed": 2.391, "read": 2.357}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 21.521, "count": 71, "max_ms": 27.358, "p95_ms": 25.506}, "gstreamer_push": {"avg_ms": 0.877, "count": 71, "max_ms": 1.213, "p95_ms": 0.921}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.459, "count": 71, "max_ms": 70.606, "p95_ms": 66.139}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.513, "count": 71, "max_ms": 102.746, "p95_ms": 101.291}, "yolo11_640_fire_11class": {"avg_ms": 53.777, "count": 71, "max_ms": 69.346, "p95_ms": 68.106}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.005, "count": 71, "max_ms": 100.718, "p95_ms": 94.506}}, "truncated_count": 0}, "processing_total": {"avg_ms": 414.535, "count": 71, "max_ms": 502.714, "p95_ms": 473.253}, "read_interval": {"avg_ms": 423.984, "count": 70, "max_ms": 513.314, "p95_ms": 483.908}, "triton_wait": {"avg_ms": 79.264, "count": 71, "max_ms": 102.757, "p95_ms": 101.304}, "visualize": {"avg_ms": 20.471, "count": 71, "max_ms": 24.899, "p95_ms": 22.493}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.7, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 20, "tracking_summary": 3055}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5171.336, "limit_mb": 8796093022207.996}, "detail": {"pid": 42443, "restart_count": 274, "start_count": 275}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 246, "h264_decode_error_count": 430, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:01:01+09:00", "last_error_summary": "[h264 @ 0x55a22bdbbf80] error while decoding MB 79 41, bytestream -33", "last_exit_at": "2026-07-02T06:01:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T06:01:04+09:00", "pid": 42443, "process": {"exists": true, "fd_count": 4, "hwm_mb": 44.008, "name": "ffmpeg", "pid": 42443, "rss_mb": 44.008, "state": "S (sleeping)", "threads": 1}, "restart_count": 274, "restart_count_per_hour": 31, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 275}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 459.714, "sample_count": 820, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 366637, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:01:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.979, "last_frame_pushed_at": "2026-07-02T06:01:01+09:00", "last_frame_read_age_sec": 5.408, "last_frame_read_at": "2026-07-02T06:01:01+09:00", "mjpeg": {"age_sec": 4.961, "exists": true, "mtime": "2026-07-02T06:01:01.791388+09:00", "mtime_epoch": 1782939661.7913883, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189370}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:01:06+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1650.719, "name": "celery", "pid": 237, "rss_mb": 1595.664, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 59, "pushed_frames": 59, "read_frames": 58, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.989, "pushed": 1.989, "read": 1.955}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 21.459, "count": 59, "max_ms": 27.358, "p95_ms": 26.321}, "gstreamer_push": {"avg_ms": 0.878, "count": 59, "max_ms": 1.213, "p95_ms": 0.928}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.674, "count": 59, "max_ms": 70.606, "p95_ms": 69.003}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.396, "count": 59, "max_ms": 102.746, "p95_ms": 101.684}, "yolo11_640_fire_11class": {"avg_ms": 53.628, "count": 59, "max_ms": 69.346, "p95_ms": 68.106}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.005, "count": 59, "max_ms": 100.718, "p95_ms": 96.541}}, "truncated_count": 0}, "processing_total": {"avg_ms": 416.141, "count": 59, "max_ms": 502.714, "p95_ms": 481.62}, "read_interval": {"avg_ms": 425.6, "count": 58, "max_ms": 513.314, "p95_ms": 492.097}, "triton_wait": {"avg_ms": 80.265, "count": 59, "max_ms": 102.757, "p95_ms": 101.695}, "visualize": {"avg_ms": 20.34, "count": 59, "max_ms": 22.531, "p95_ms": 22.223}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.67, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 3, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 33, "tracking_summary": 3057}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5263.195, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5648a1825a00] error while decoding MB 40 39, bytestream -15", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 247, "h264_decode_error_count": 431, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:03:28+09:00", "last_error_summary": "[h264 @ 0x5648a1825a00] error while decoding MB 40 39, bytestream -15", "last_exit_at": "2026-07-02T06:01:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 144.223, "last_started_at": "2026-07-02T06:01:04+09:00", "pid": 42443, "process": {"exists": false, "pid": 42443}, "restart_count": 274, "restart_count_per_hour": 30, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 275}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 558.273, "sample_count": 635, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 366921, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:03:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-02T06:03:28+09:00", "last_frame_read_age_sec": 0.048, "last_frame_read_at": "2026-07-02T06:03:28+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-02T06:03:28.934997+09:00", "mtime_epoch": 1782939808.9349966, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194170}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:03:28+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1670.121, "name": "celery", "pid": 237, "rss_mb": 1670.121, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 58, "pushed_frames": 58, "read_frames": 58, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.961, "pushed": 1.961, "read": 1.961}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.878, "count": 58, "max_ms": 32.535, "p95_ms": 30.248}, "gstreamer_push": {"avg_ms": 0.954, "count": 58, "max_ms": 3.005, "p95_ms": 1.854}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.44, "count": 58, "max_ms": 71.655, "p95_ms": 65.33}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.869, "count": 58, "max_ms": 104.781, "p95_ms": 101.933}, "yolo11_640_fire_11class": {"avg_ms": 53.251, "count": 58, "max_ms": 70.151, "p95_ms": 69.802}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.858, "count": 58, "max_ms": 104.77, "p95_ms": 100.405}}, "truncated_count": 0}, "processing_total": {"avg_ms": 507.141, "count": 58, "max_ms": 712.803, "p95_ms": 691.311}, "read_interval": {"avg_ms": 516.975, "count": 58, "max_ms": 722.821, "p95_ms": 703.535}, "triton_wait": {"avg_ms": 81.883, "count": 58, "max_ms": 104.79, "p95_ms": 101.945}, "visualize": {"avg_ms": 24.092, "count": 58, "max_ms": 38.383, "p95_ms": 32.018}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.571, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 3, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 33, "tracking_summary": 3057}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5261.73, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T06:03:28+09:00", "last_error_summary": "[h264 @ 0x5648a1825a00] error while decoding MB 40 39, bytestream -15", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 247, "h264_decode_error_count": 431, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:03:28+09:00", "last_error_summary": "[h264 @ 0x5648a1825a00] error while decoding MB 40 39, bytestream -15", "last_exit_at": "2026-07-02T06:03:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 144.723, "last_started_at": "2026-07-02T06:01:04+09:00", "pid": 42443, "process": {"exists": false, "pid": 42443}, "restart_count": 274, "restart_count_per_hour": 30, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 275}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 558.273, "sample_count": 635, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 366922, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:03:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T06:03:29+09:00", "last_frame_read_age_sec": 0.549, "last_frame_read_at": "2026-07-02T06:03:28+09:00", "mjpeg": {"age_sec": 0.539, "exists": true, "mtime": "2026-07-02T06:03:28.934997+09:00", "mtime_epoch": 1782939808.9349966, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194170}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:03:29+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1670.121, "name": "celery", "pid": 237, "rss_mb": 1670.121, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 58, "pushed_frames": 58, "read_frames": 57, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.961, "pushed": 1.961, "read": 1.927}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.99, "count": 58, "max_ms": 32.535, "p95_ms": 30.248}, "gstreamer_push": {"avg_ms": 0.954, "count": 58, "max_ms": 3.005, "p95_ms": 1.854}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.456, "count": 58, "max_ms": 71.655, "p95_ms": 65.33}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.754, "count": 58, "max_ms": 104.781, "p95_ms": 101.933}, "yolo11_640_fire_11class": {"avg_ms": 53.25, "count": 58, "max_ms": 70.151, "p95_ms": 69.802}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.862, "count": 58, "max_ms": 104.77, "p95_ms": 100.405}}, "truncated_count": 0}, "processing_total": {"avg_ms": 508.576, "count": 58, "max_ms": 712.803, "p95_ms": 691.311}, "read_interval": {"avg_ms": 517.74, "count": 57, "max_ms": 722.821, "p95_ms": 703.535}, "triton_wait": {"avg_ms": 81.768, "count": 58, "max_ms": 104.79, "p95_ms": 101.945}, "visualize": {"avg_ms": 24.316, "count": 58, "max_ms": 38.383, "p95_ms": 32.694}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.58, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 3, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 33, "tracking_summary": 3057}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5261.707, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 247, "h264_decode_error_count": 431, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:03:28+09:00", "last_error_summary": "[h264 @ 0x5648a1825a00] error while decoding MB 40 39, bytestream -15", "last_exit_at": "2026-07-02T06:03:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 144.726, "last_started_at": "2026-07-02T06:01:04+09:00", "pid": 42443, "process": {"exists": false, "pid": 42443}, "restart_count": 274, "restart_count_per_hour": 30, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 275}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 558.273, "sample_count": 635, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 366922, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:03:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T06:03:29+09:00", "last_frame_read_age_sec": 0.552, "last_frame_read_at": "2026-07-02T06:03:28+09:00", "mjpeg": {"age_sec": 0.542, "exists": true, "mtime": "2026-07-02T06:03:28.934997+09:00", "mtime_epoch": 1782939808.9349966, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194170}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:03:29+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1670.121, "name": "celery", "pid": 237, "rss_mb": 1670.121, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 58, "pushed_frames": 58, "read_frames": 57, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.961, "pushed": 1.961, "read": 1.927}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.99, "count": 58, "max_ms": 32.535, "p95_ms": 30.248}, "gstreamer_push": {"avg_ms": 0.954, "count": 58, "max_ms": 3.005, "p95_ms": 1.854}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.456, "count": 58, "max_ms": 71.655, "p95_ms": 65.33}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.754, "count": 58, "max_ms": 104.781, "p95_ms": 101.933}, "yolo11_640_fire_11class": {"avg_ms": 53.25, "count": 58, "max_ms": 70.151, "p95_ms": 69.802}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.862, "count": 58, "max_ms": 104.77, "p95_ms": 100.405}}, "truncated_count": 0}, "processing_total": {"avg_ms": 508.576, "count": 58, "max_ms": 712.803, "p95_ms": 691.311}, "read_interval": {"avg_ms": 517.74, "count": 57, "max_ms": 722.821, "p95_ms": 703.535}, "triton_wait": {"avg_ms": 81.768, "count": 58, "max_ms": 104.79, "p95_ms": 101.945}, "visualize": {"avg_ms": 24.316, "count": 58, "max_ms": 38.383, "p95_ms": 32.694}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.583, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 3, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 33, "tracking_summary": 3057}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5370.18, "limit_mb": 8796093022207.996}, "detail": {"pid": 42670, "restart_count": 275, "start_count": 276}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 247, "h264_decode_error_count": 431, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:03:28+09:00", "last_error_summary": "[h264 @ 0x5648a1825a00] error while decoding MB 40 39, bytestream -15", "last_exit_at": "2026-07-02T06:03:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T06:03:32+09:00", "pid": 42670, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.238, "name": "ffmpeg", "pid": 42670, "rss_mb": 139.238, "state": "S (sleeping)", "threads": 106}, "restart_count": 275, "restart_count_per_hour": 31, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 276}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 559.087, "sample_count": 619, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 366922, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:03:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.889, "last_frame_pushed_at": "2026-07-02T06:03:29+09:00", "last_frame_read_age_sec": 5.436, "last_frame_read_at": "2026-07-02T06:03:28+09:00", "mjpeg": {"age_sec": 4.87, "exists": true, "mtime": "2026-07-02T06:03:29.491010+09:00", "mtime_epoch": 1782939809.49101, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194816}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:03:34+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1670.121, "name": "celery", "pid": 237, "rss_mb": 1670.121, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 48, "pushed_frames": 48, "read_frames": 47, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.604, "pushed": 1.604, "read": 1.57}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 26.804, "count": 47, "max_ms": 32.535, "p95_ms": 30.248}, "gstreamer_push": {"avg_ms": 0.974, "count": 48, "max_ms": 3.005, "p95_ms": 1.854}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.438, "count": 47, "max_ms": 71.655, "p95_ms": 65.33}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.42, "count": 47, "max_ms": 104.422, "p95_ms": 100.738}, "yolo11_640_fire_11class": {"avg_ms": 53.221, "count": 47, "max_ms": 69.802, "p95_ms": 68.763}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.886, "count": 47, "max_ms": 102.244, "p95_ms": 99.794}}, "truncated_count": 0}, "processing_total": {"avg_ms": 521.193, "count": 48, "max_ms": 712.803, "p95_ms": 691.311}, "read_interval": {"avg_ms": 530.398, "count": 47, "max_ms": 722.821, "p95_ms": 703.535}, "triton_wait": {"avg_ms": 81.433, "count": 47, "max_ms": 104.434, "p95_ms": 100.75}, "visualize": {"avg_ms": 25.338, "count": 47, "max_ms": 38.383, "p95_ms": 32.694}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.929, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 40, "tracking_summary": 3060}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5252.836, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55db0fab2200] error while decoding MB 40 24, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 248, "h264_decode_error_count": 432, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:05:34+09:00", "last_error_summary": "[h264 @ 0x55db0fab2200] error while decoding MB 40 24, bytestream -5", "last_exit_at": "2026-07-02T06:03:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 122.524, "last_started_at": "2026-07-02T06:03:32+09:00", "pid": 42670, "process": {"exists": false, "pid": 42670}, "restart_count": 275, "restart_count_per_hour": 30, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 276}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 539.992, "sample_count": 714, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 367287, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:05:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-02T06:05:34+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-02T06:05:34+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-02T06:05:34.842003+09:00", "mtime_epoch": 1782939934.8420033, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189350}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:05:34+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1670.121, "name": "celery", "pid": 237, "rss_mb": 1650.863, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 94, "pushed_frames": 94, "read_frames": 94, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.14, "pushed": 3.14, "read": 3.14}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.975, "count": 93, "max_ms": 18.559, "p95_ms": 17.037}, "gstreamer_push": {"avg_ms": 0.871, "count": 94, "max_ms": 1.197, "p95_ms": 0.922}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.828, "count": 93, "max_ms": 64.702, "p95_ms": 62.637}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.023, "count": 93, "max_ms": 100.035, "p95_ms": 93.199}, "yolo11_640_fire_11class": {"avg_ms": 53.596, "count": 93, "max_ms": 78.13, "p95_ms": 67.494}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.375, "count": 93, "max_ms": 101.203, "p95_ms": 90.281}}, "truncated_count": 0}, "processing_total": {"avg_ms": 311.611, "count": 94, "max_ms": 358.001, "p95_ms": 349.334}, "read_interval": {"avg_ms": 321.162, "count": 94, "max_ms": 367.195, "p95_ms": 359.665}, "triton_wait": {"avg_ms": 73.455, "count": 93, "max_ms": 101.212, "p95_ms": 94.02}, "visualize": {"avg_ms": 13.503, "count": 93, "max_ms": 16.231, "p95_ms": 15.189}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.937, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 40, "tracking_summary": 3060}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5251.316, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T06:05:34+09:00", "last_error_summary": "[h264 @ 0x55db0fab2200] error while decoding MB 40 24, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 248, "h264_decode_error_count": 432, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:05:34+09:00", "last_error_summary": "[h264 @ 0x55db0fab2200] error while decoding MB 40 24, bytestream -5", "last_exit_at": "2026-07-02T06:05:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 122.808, "last_started_at": "2026-07-02T06:03:32+09:00", "pid": 42670, "process": {"exists": false, "pid": 42670}, "restart_count": 275, "restart_count_per_hour": 30, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 276}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 539.992, "sample_count": 714, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 367288, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:05:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T06:05:35+09:00", "last_frame_read_age_sec": 0.334, "last_frame_read_at": "2026-07-02T06:05:34+09:00", "mjpeg": {"age_sec": 0.327, "exists": true, "mtime": "2026-07-02T06:05:34.842003+09:00", "mtime_epoch": 1782939934.8420033, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189350}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:05:35+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1670.121, "name": "celery", "pid": 237, "rss_mb": 1650.863, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 94, "pushed_frames": 94, "read_frames": 93, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.14, "pushed": 3.14, "read": 3.107}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.968, "count": 93, "max_ms": 18.559, "p95_ms": 17.037}, "gstreamer_push": {"avg_ms": 0.871, "count": 94, "max_ms": 1.197, "p95_ms": 0.922}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.838, "count": 93, "max_ms": 64.702, "p95_ms": 62.637}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.226, "count": 93, "max_ms": 100.035, "p95_ms": 93.368}, "yolo11_640_fire_11class": {"avg_ms": 53.519, "count": 93, "max_ms": 78.13, "p95_ms": 67.494}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.57, "count": 93, "max_ms": 101.203, "p95_ms": 90.349}}, "truncated_count": 0}, "processing_total": {"avg_ms": 311.907, "count": 94, "max_ms": 358.001, "p95_ms": 349.334}, "read_interval": {"avg_ms": 321.253, "count": 93, "max_ms": 367.195, "p95_ms": 359.665}, "triton_wait": {"avg_ms": 73.658, "count": 93, "max_ms": 101.212, "p95_ms": 94.02}, "visualize": {"avg_ms": 13.534, "count": 93, "max_ms": 16.231, "p95_ms": 15.189}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.936, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 40, "tracking_summary": 3060}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5251.262, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 248, "h264_decode_error_count": 432, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:05:34+09:00", "last_error_summary": "[h264 @ 0x55db0fab2200] error while decoding MB 40 24, bytestream -5", "last_exit_at": "2026-07-02T06:05:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 122.811, "last_started_at": "2026-07-02T06:03:32+09:00", "pid": 42670, "process": {"exists": false, "pid": 42670}, "restart_count": 275, "restart_count_per_hour": 30, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 276}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 539.992, "sample_count": 714, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 367288, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:05:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T06:05:35+09:00", "last_frame_read_age_sec": 0.337, "last_frame_read_at": "2026-07-02T06:05:34+09:00", "mjpeg": {"age_sec": 0.33, "exists": true, "mtime": "2026-07-02T06:05:34.842003+09:00", "mtime_epoch": 1782939934.8420033, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189350}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:05:35+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1670.121, "name": "celery", "pid": 237, "rss_mb": 1650.863, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 94, "pushed_frames": 94, "read_frames": 93, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.14, "pushed": 3.14, "read": 3.106}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.968, "count": 93, "max_ms": 18.559, "p95_ms": 17.037}, "gstreamer_push": {"avg_ms": 0.871, "count": 94, "max_ms": 1.197, "p95_ms": 0.922}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.838, "count": 93, "max_ms": 64.702, "p95_ms": 62.637}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.226, "count": 93, "max_ms": 100.035, "p95_ms": 93.368}, "yolo11_640_fire_11class": {"avg_ms": 53.519, "count": 93, "max_ms": 78.13, "p95_ms": 67.494}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.57, "count": 93, "max_ms": 101.203, "p95_ms": 90.349}}, "truncated_count": 0}, "processing_total": {"avg_ms": 311.907, "count": 94, "max_ms": 358.001, "p95_ms": 349.334}, "read_interval": {"avg_ms": 321.253, "count": 93, "max_ms": 367.195, "p95_ms": 359.665}, "triton_wait": {"avg_ms": 73.658, "count": 93, "max_ms": 101.212, "p95_ms": 94.02}, "visualize": {"avg_ms": 13.534, "count": 93, "max_ms": 16.231, "p95_ms": 15.189}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.939, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 40, "tracking_summary": 3060}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5359.68, "limit_mb": 8796093022207.996}, "detail": {"pid": 42777, "restart_count": 276, "start_count": 277}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 248, "h264_decode_error_count": 432, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:05:34+09:00", "last_error_summary": "[h264 @ 0x55db0fab2200] error while decoding MB 40 24, bytestream -5", "last_exit_at": "2026-07-02T06:05:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T06:05:37+09:00", "pid": 42777, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.184, "name": "ffmpeg", "pid": 42777, "rss_mb": 139.184, "state": "S (sleeping)", "threads": 106}, "restart_count": 276, "restart_count_per_hour": 31, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 277}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 540.552, "sample_count": 703, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 367288, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:05:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.383, "last_frame_pushed_at": "2026-07-02T06:05:35+09:00", "last_frame_read_age_sec": 4.714, "last_frame_read_at": "2026-07-02T06:05:34+09:00", "mjpeg": {"age_sec": 4.367, "exists": true, "mtime": "2026-07-02T06:05:35.182011+09:00", "mtime_epoch": 1782939935.1820114, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190123}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:05:39+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1670.121, "name": "celery", "pid": 237, "rss_mb": 1650.863, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 79, "pushed_frames": 79, "read_frames": 78, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.663, "pushed": 2.663, "read": 2.63}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 15.072, "count": 79, "max_ms": 18.559, "p95_ms": 17.058}, "gstreamer_push": {"avg_ms": 0.869, "count": 79, "max_ms": 1.045, "p95_ms": 0.922}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.099, "count": 79, "max_ms": 64.702, "p95_ms": 60.983}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.973, "count": 79, "max_ms": 99.303, "p95_ms": 93.199}, "yolo11_640_fire_11class": {"avg_ms": 52.805, "count": 79, "max_ms": 78.13, "p95_ms": 67.382}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.437, "count": 79, "max_ms": 101.203, "p95_ms": 91.915}}, "truncated_count": 0}, "processing_total": {"avg_ms": 315.101, "count": 79, "max_ms": 358.001, "p95_ms": 349.334}, "read_interval": {"avg_ms": 324.407, "count": 78, "max_ms": 367.195, "p95_ms": 359.665}, "triton_wait": {"avg_ms": 73.149, "count": 79, "max_ms": 101.212, "p95_ms": 94.02}, "visualize": {"avg_ms": 13.721, "count": 79, "max_ms": 16.231, "p95_ms": 15.189}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.66, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 47, "tracking_summary": 3063}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5419.395, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T06:05:34+09:00", "last_error_summary": "[h264 @ 0x55db0fab2200] error while decoding MB 40 24, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 248, "h264_decode_error_count": 432, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:05:34+09:00", "last_error_summary": "[h264 @ 0x55db0fab2200] error while decoding MB 40 24, bytestream -5", "last_exit_at": "2026-07-02T06:07:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.936, "last_started_at": "2026-07-02T06:05:37+09:00", "pid": 42777, "process": {"exists": false, "pid": 42777}, "restart_count": 276, "restart_count_per_hour": 30, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 277}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 499.765, "sample_count": 751, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 367586, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:07:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T06:07:46+09:00", "last_frame_read_age_sec": 0.474, "last_frame_read_at": "2026-07-02T06:07:46+09:00", "mjpeg": {"age_sec": 0.467, "exists": true, "mtime": "2026-07-02T06:07:46.017075+09:00", "mtime_epoch": 1782940066.0170753, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194097}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:07:46+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1670.121, "name": "celery", "pid": 237, "rss_mb": 1651.121, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 7.665, "recorders": [{"age_sec": 7.665, "frame_count": 66, "is_recording": true, "is_stopping": false, "pid": 42944, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.77, "name": "ffmpeg", "pid": 42944, "rss_mb": 186.77, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 34, "returncode": null, "running": true, "started_at": "2026-07-02T06:07:38+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_060738_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [42944], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 73, "pushed_frames": 73, "read_frames": 72, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.444, "pushed": 2.444, "read": 2.41}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 20.611, "count": 72, "max_ms": 27.455, "p95_ms": 25.538}, "gstreamer_push": {"avg_ms": 0.883, "count": 73, "max_ms": 1.276, "p95_ms": 0.961}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.679, "count": 72, "max_ms": 80.42, "p95_ms": 69.501}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.202, "count": 72, "max_ms": 105.173, "p95_ms": 95.958}, "yolo11_640_fire_11class": {"avg_ms": 54.201, "count": 72, "max_ms": 72.728, "p95_ms": 68.511}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.243, "count": 72, "max_ms": 102.741, "p95_ms": 92.538}}, "truncated_count": 0}, "processing_total": {"avg_ms": 405.148, "count": 73, "max_ms": 1019.084, "p95_ms": 475.731}, "read_interval": {"avg_ms": 413.974, "count": 72, "max_ms": 1029.598, "p95_ms": 486.045}, "triton_wait": {"avg_ms": 79.216, "count": 72, "max_ms": 105.185, "p95_ms": 95.971}, "visualize": {"avg_ms": 18.198, "count": 72, "max_ms": 29.658, "p95_ms": 28.103}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.872, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 47, "tracking_summary": 3063}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5419.387, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 248, "h264_decode_error_count": 432, "h264_decode_error_count_per_hour": 30, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:05:34+09:00", "last_error_summary": "[h264 @ 0x55db0fab2200] error while decoding MB 40 24, bytestream -5", "last_exit_at": "2026-07-02T06:07:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.939, "last_started_at": "2026-07-02T06:05:37+09:00", "pid": 42777, "process": {"exists": false, "pid": 42777}, "restart_count": 276, "restart_count_per_hour": 30, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 277}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 499.765, "sample_count": 751, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 367586, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:07:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-02T06:07:46+09:00", "last_frame_read_age_sec": 0.477, "last_frame_read_at": "2026-07-02T06:07:46+09:00", "mjpeg": {"age_sec": 0.47, "exists": true, "mtime": "2026-07-02T06:07:46.017075+09:00", "mtime_epoch": 1782940066.0170753, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194097}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:07:46+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1670.121, "name": "celery", "pid": 237, "rss_mb": 1651.121, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 7.668, "recorders": [{"age_sec": 7.668, "frame_count": 66, "is_recording": true, "is_stopping": false, "pid": 42944, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.77, "name": "ffmpeg", "pid": 42944, "rss_mb": 186.77, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 34, "returncode": null, "running": true, "started_at": "2026-07-02T06:07:38+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_060738_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [42944], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 73, "pushed_frames": 73, "read_frames": 72, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.444, "pushed": 2.444, "read": 2.41}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 20.611, "count": 72, "max_ms": 27.455, "p95_ms": 25.538}, "gstreamer_push": {"avg_ms": 0.883, "count": 73, "max_ms": 1.276, "p95_ms": 0.961}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.679, "count": 72, "max_ms": 80.42, "p95_ms": 69.501}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.202, "count": 72, "max_ms": 105.173, "p95_ms": 95.958}, "yolo11_640_fire_11class": {"avg_ms": 54.201, "count": 72, "max_ms": 72.728, "p95_ms": 68.511}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.243, "count": 72, "max_ms": 102.741, "p95_ms": 92.538}}, "truncated_count": 0}, "processing_total": {"avg_ms": 405.148, "count": 73, "max_ms": 1019.084, "p95_ms": 475.731}, "read_interval": {"avg_ms": 413.974, "count": 72, "max_ms": 1029.598, "p95_ms": 486.045}, "triton_wait": {"avg_ms": 79.216, "count": 72, "max_ms": 105.185, "p95_ms": 95.971}, "visualize": {"avg_ms": 18.198, "count": 72, "max_ms": 29.658, "p95_ms": 28.103}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.875, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 47, "tracking_summary": 3063}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5525.898, "limit_mb": 8796093022207.996}, "detail": {"pid": 43004, "restart_count": 277, "start_count": 278}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 248, "h264_decode_error_count": 432, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:05:34+09:00", "last_error_summary": "[h264 @ 0x55db0fab2200] error while decoding MB 40 24, bytestream -5", "last_exit_at": "2026-07-02T06:07:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T06:07:49+09:00", "pid": 43004, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.781, "name": "ffmpeg", "pid": 43004, "rss_mb": 136.781, "state": "S (sleeping)", "threads": 106}, "restart_count": 277, "restart_count_per_hour": 31, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 278}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 500.935, "sample_count": 740, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 367586, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:07:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.832, "last_frame_pushed_at": "2026-07-02T06:07:46+09:00", "last_frame_read_age_sec": 5.301, "last_frame_read_at": "2026-07-02T06:07:46+09:00", "mjpeg": {"age_sec": 4.814, "exists": true, "mtime": "2026-07-02T06:07:46.497087+09:00", "mtime_epoch": 1782940066.4970865, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195444}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:07:51+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 1670.121, "name": "celery", "pid": 237, "rss_mb": 1651.121, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 12.494, "recorders": [{"age_sec": 12.494, "frame_count": 66, "is_recording": true, "is_stopping": false, "pid": 42944, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.77, "name": "ffmpeg", "pid": 42944, "rss_mb": 186.77, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 34, "returncode": null, "running": true, "started_at": "2026-07-02T06:07:38+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_060738_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [42944], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 62, "pushed_frames": 62, "read_frames": 61, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.072, "pushed": 2.072, "read": 2.038}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 19.961, "count": 61, "max_ms": 23.839, "p95_ms": 22.294}, "gstreamer_push": {"avg_ms": 0.886, "count": 62, "max_ms": 1.276, "p95_ms": 0.961}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.582, "count": 61, "max_ms": 80.42, "p95_ms": 68.496}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.42, "count": 61, "max_ms": 105.173, "p95_ms": 95.589}, "yolo11_640_fire_11class": {"avg_ms": 53.944, "count": 61, "max_ms": 72.728, "p95_ms": 68.511}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.276, "count": 61, "max_ms": 102.741, "p95_ms": 91.387}}, "truncated_count": 0}, "processing_total": {"avg_ms": 401.299, "count": 62, "max_ms": 1019.084, "p95_ms": 468.207}, "read_interval": {"avg_ms": 409.945, "count": 61, "max_ms": 1029.598, "p95_ms": 478.714}, "triton_wait": {"avg_ms": 78.433, "count": 61, "max_ms": 105.185, "p95_ms": 95.602}, "visualize": {"avg_ms": 18.761, "count": 61, "max_ms": 29.658, "p95_ms": 28.103}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.925, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 2, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 70, "tracking_summary": 3065}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6156.145, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55afabbe0b40] error while decoding MB 25 41, bytestream -11", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 249, "h264_decode_error_count": 433, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:10:26+09:00", "last_error_summary": "[h264 @ 0x55afabbe0b40] error while decoding MB 25 41, bytestream -11", "last_exit_at": "2026-07-02T06:07:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 156.721, "last_started_at": "2026-07-02T06:07:49+09:00", "pid": 43004, "process": {"exists": false, "pid": 43004}, "restart_count": 277, "restart_count_per_hour": 29, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 278}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 750.82, "sample_count": 577, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 367836, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:10:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.071, "last_frame_pushed_at": "2026-07-02T06:10:25+09:00", "last_frame_read_age_sec": 0.062, "last_frame_read_at": "2026-07-02T06:10:25+09:00", "mjpeg": {"age_sec": 0.054, "exists": true, "mtime": "2026-07-02T06:10:25.976757+09:00", "mtime_epoch": 1782940225.9767573, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195854}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:10:26+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2584.477, "name": "celery", "pid": 237, "rss_mb": 2541.582, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 39, "pushed_frames": 39, "read_frames": 39, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.335, "pushed": 1.335, "read": 1.335}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 41.938, "count": 39, "max_ms": 52.557, "p95_ms": 46.946}, "gstreamer_push": {"avg_ms": 1.001, "count": 39, "max_ms": 1.891, "p95_ms": 1.885}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.864, "count": 39, "max_ms": 78.248, "p95_ms": 77.925}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 86.924, "count": 39, "max_ms": 110.734, "p95_ms": 107.26}, "yolo11_640_fire_11class": {"avg_ms": 53.56, "count": 39, "max_ms": 71.361, "p95_ms": 70.84}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.523, "count": 39, "max_ms": 104.206, "p95_ms": 103.289}}, "truncated_count": 0}, "processing_total": {"avg_ms": 759.947, "count": 39, "max_ms": 1124.517, "p95_ms": 1005.641}, "read_interval": {"avg_ms": 769.658, "count": 39, "max_ms": 1134.574, "p95_ms": 1017.873}, "triton_wait": {"avg_ms": 87.059, "count": 39, "max_ms": 110.748, "p95_ms": 107.274}, "visualize": {"avg_ms": 27.78, "count": 39, "max_ms": 33.915, "p95_ms": 33.738}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.207, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 2, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 70, "tracking_summary": 3065}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6154.758, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T06:10:26+09:00", "last_error_summary": "[h264 @ 0x55afabbe0b40] error while decoding MB 25 41, bytestream -11", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 249, "h264_decode_error_count": 433, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:10:26+09:00", "last_error_summary": "[h264 @ 0x55afabbe0b40] error while decoding MB 25 41, bytestream -11", "last_exit_at": "2026-07-02T06:10:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 157.288, "last_started_at": "2026-07-02T06:07:49+09:00", "pid": 43004, "process": {"exists": false, "pid": 43004}, "restart_count": 277, "restart_count_per_hour": 29, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 278}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 750.82, "sample_count": 577, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 367837, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:10:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.002, "last_frame_pushed_at": "2026-07-02T06:10:26+09:00", "last_frame_read_age_sec": 0.629, "last_frame_read_at": "2026-07-02T06:10:25+09:00", "mjpeg": {"age_sec": 0.622, "exists": true, "mtime": "2026-07-02T06:10:25.976757+09:00", "mtime_epoch": 1782940225.9767573, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195854}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:10:26+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 2584.477, "name": "celery", "pid": 237, "rss_mb": 2541.582, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 40, "pushed_frames": 40, "read_frames": 39, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.343, "pushed": 1.343, "read": 1.31}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 41.635, "count": 39, "max_ms": 52.557, "p95_ms": 46.366}, "gstreamer_push": {"avg_ms": 0.998, "count": 40, "max_ms": 1.891, "p95_ms": 1.861}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.153, "count": 39, "max_ms": 78.248, "p95_ms": 77.925}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 86.465, "count": 39, "max_ms": 110.734, "p95_ms": 107.26}, "yolo11_640_fire_11class": {"avg_ms": 53.141, "count": 39, "max_ms": 70.84, "p95_ms": 68.907}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.371, "count": 39, "max_ms": 104.206, "p95_ms": 103.289}}, "truncated_count": 0}, "processing_total": {"avg_ms": 756.591, "count": 40, "max_ms": 1124.517, "p95_ms": 955.731}, "read_interval": {"avg_ms": 769.658, "count": 39, "max_ms": 1134.574, "p95_ms": 1017.873}, "triton_wait": {"avg_ms": 86.6, "count": 39, "max_ms": 110.748, "p95_ms": 107.274}, "visualize": {"avg_ms": 27.785, "count": 39, "max_ms": 33.915, "p95_ms": 33.738}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.774, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 2, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 70, "tracking_summary": 3065}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6154.875, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 249, "h264_decode_error_count": 433, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:10:26+09:00", "last_error_summary": "[h264 @ 0x55afabbe0b40] error while decoding MB 25 41, bytestream -11", "last_exit_at": "2026-07-02T06:10:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 157.291, "last_started_at": "2026-07-02T06:07:49+09:00", "pid": 43004, "process": {"exists": false, "pid": 43004}, "restart_count": 277, "restart_count_per_hour": 29, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 278}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 750.82, "sample_count": 576, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 367837, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:10:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T06:10:26+09:00", "last_frame_read_age_sec": 0.632, "last_frame_read_at": "2026-07-02T06:10:25+09:00", "mjpeg": {"age_sec": 0.625, "exists": true, "mtime": "2026-07-02T06:10:25.976757+09:00", "mtime_epoch": 1782940225.9767573, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195854}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:10:26+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 2584.477, "name": "celery", "pid": 237, "rss_mb": 2541.582, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 40, "pushed_frames": 40, "read_frames": 39, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.343, "pushed": 1.343, "read": 1.31}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 41.635, "count": 39, "max_ms": 52.557, "p95_ms": 46.366}, "gstreamer_push": {"avg_ms": 0.998, "count": 40, "max_ms": 1.891, "p95_ms": 1.861}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.153, "count": 39, "max_ms": 78.248, "p95_ms": 77.925}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 86.465, "count": 39, "max_ms": 110.734, "p95_ms": 107.26}, "yolo11_640_fire_11class": {"avg_ms": 53.141, "count": 39, "max_ms": 70.84, "p95_ms": 68.907}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.371, "count": 39, "max_ms": 104.206, "p95_ms": 103.289}}, "truncated_count": 0}, "processing_total": {"avg_ms": 756.591, "count": 40, "max_ms": 1124.517, "p95_ms": 955.731}, "read_interval": {"avg_ms": 769.658, "count": 39, "max_ms": 1134.574, "p95_ms": 1017.873}, "triton_wait": {"avg_ms": 86.6, "count": 39, "max_ms": 110.748, "p95_ms": 107.274}, "visualize": {"avg_ms": 27.785, "count": 39, "max_ms": 33.915, "p95_ms": 33.738}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.777, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 2, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 70, "tracking_summary": 3065}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 6259.969, "limit_mb": 8796093022207.996}, "detail": {"pid": 43171, "restart_count": 278, "start_count": 279}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 249, "h264_decode_error_count": 433, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:10:26+09:00", "last_error_summary": "[h264 @ 0x55afabbe0b40] error while decoding MB 25 41, bytestream -11", "last_exit_at": "2026-07-02T06:10:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T06:10:29+09:00", "pid": 43171, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.879, "name": "ffmpeg", "pid": 43171, "rss_mb": 136.879, "state": "S (sleeping)", "threads": 106}, "restart_count": 278, "restart_count_per_hour": 30, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 279}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 752.312, "sample_count": 562, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 367837, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:10:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.732, "last_frame_pushed_at": "2026-07-02T06:10:26+09:00", "last_frame_read_age_sec": 5.359, "last_frame_read_at": "2026-07-02T06:10:25+09:00", "mjpeg": {"age_sec": 4.715, "exists": true, "mtime": "2026-07-02T06:10:26.612772+09:00", "mtime_epoch": 1782940226.6127717, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195334}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:10:31+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2584.477, "name": "celery", "pid": 237, "rss_mb": 2541.582, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 35, "pushed_frames": 35, "read_frames": 34, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.169, "pushed": 1.169, "read": 1.136}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 41.153, "count": 34, "max_ms": 52.557, "p95_ms": 46.366}, "gstreamer_push": {"avg_ms": 0.955, "count": 35, "max_ms": 1.885, "p95_ms": 1.861}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.432, "count": 34, "max_ms": 78.248, "p95_ms": 77.925}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 86.196, "count": 34, "max_ms": 110.734, "p95_ms": 107.26}, "yolo11_640_fire_11class": {"avg_ms": 53.265, "count": 34, "max_ms": 70.84, "p95_ms": 68.907}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.431, "count": 34, "max_ms": 104.206, "p95_ms": 103.289}}, "truncated_count": 0}, "processing_total": {"avg_ms": 732.563, "count": 35, "max_ms": 1005.641, "p95_ms": 913.175}, "read_interval": {"avg_ms": 745.397, "count": 34, "max_ms": 1017.873, "p95_ms": 921.796}, "triton_wait": {"avg_ms": 86.349, "count": 34, "max_ms": 110.748, "p95_ms": 107.274}, "visualize": {"avg_ms": 27.718, "count": 34, "max_ms": 33.915, "p95_ms": 33.738}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.932, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 77, "tracking_summary": 3067}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5545.402, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T06:10:26+09:00", "last_error_summary": "[h264 @ 0x55afabbe0b40] error while decoding MB 25 41, bytestream -11", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 249, "h264_decode_error_count": 433, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:10:26+09:00", "last_error_summary": "[h264 @ 0x55afabbe0b40] error while decoding MB 25 41, bytestream -11", "last_exit_at": "2026-07-02T06:12:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.856, "last_started_at": "2026-07-02T06:10:29+09:00", "pid": 43171, "process": {"exists": false, "pid": 43171}, "restart_count": 278, "restart_count_per_hour": 29, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 279}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 750.82, "sample_count": 556, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 368130, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:12:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T06:12:41+09:00", "last_frame_read_age_sec": 0.365, "last_frame_read_at": "2026-07-02T06:12:40+09:00", "mjpeg": {"age_sec": 0.356, "exists": true, "mtime": "2026-07-02T06:12:40.827820+09:00", "mtime_epoch": 1782940360.82782, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194531}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:12:41+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 2584.477, "name": "celery", "pid": 237, "rss_mb": 1918.656, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 74, "pushed_frames": 74, "read_frames": 73, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.47, "pushed": 2.47, "read": 2.437}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 22.573, "count": 73, "max_ms": 33.899, "p95_ms": 27.703}, "gstreamer_push": {"avg_ms": 0.875, "count": 74, "max_ms": 1.024, "p95_ms": 0.926}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.471, "count": 73, "max_ms": 70.887, "p95_ms": 65.346}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.708, "count": 73, "max_ms": 110.148, "p95_ms": 104.199}, "yolo11_640_fire_11class": {"avg_ms": 53.496, "count": 73, "max_ms": 74.239, "p95_ms": 68.816}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.841, "count": 73, "max_ms": 106.455, "p95_ms": 90.671}}, "truncated_count": 0}, "processing_total": {"avg_ms": 401.907, "count": 74, "max_ms": 531.119, "p95_ms": 481.547}, "read_interval": {"avg_ms": 411.876, "count": 73, "max_ms": 539.997, "p95_ms": 490.428}, "triton_wait": {"avg_ms": 82.82, "count": 73, "max_ms": 110.16, "p95_ms": 104.211}, "visualize": {"avg_ms": 18.314, "count": 73, "max_ms": 26.055, "p95_ms": 23.001}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.954, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 77, "tracking_summary": 3067}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5545.395, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 249, "h264_decode_error_count": 433, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:10:26+09:00", "last_error_summary": "[h264 @ 0x55afabbe0b40] error while decoding MB 25 41, bytestream -11", "last_exit_at": "2026-07-02T06:12:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.859, "last_started_at": "2026-07-02T06:10:29+09:00", "pid": 43171, "process": {"exists": false, "pid": 43171}, "restart_count": 278, "restart_count_per_hour": 29, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 279}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 750.82, "sample_count": 556, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 368130, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:12:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T06:12:41+09:00", "last_frame_read_age_sec": 0.368, "last_frame_read_at": "2026-07-02T06:12:40+09:00", "mjpeg": {"age_sec": 0.358, "exists": true, "mtime": "2026-07-02T06:12:40.827820+09:00", "mtime_epoch": 1782940360.82782, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194531}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:12:41+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 2584.477, "name": "celery", "pid": 237, "rss_mb": 1918.656, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 74, "pushed_frames": 74, "read_frames": 73, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.47, "pushed": 2.47, "read": 2.437}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 22.573, "count": 73, "max_ms": 33.899, "p95_ms": 27.703}, "gstreamer_push": {"avg_ms": 0.875, "count": 74, "max_ms": 1.024, "p95_ms": 0.926}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.471, "count": 73, "max_ms": 70.887, "p95_ms": 65.346}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.708, "count": 73, "max_ms": 110.148, "p95_ms": 104.199}, "yolo11_640_fire_11class": {"avg_ms": 53.496, "count": 73, "max_ms": 74.239, "p95_ms": 68.816}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.841, "count": 73, "max_ms": 106.455, "p95_ms": 90.671}}, "truncated_count": 0}, "processing_total": {"avg_ms": 401.907, "count": 74, "max_ms": 531.119, "p95_ms": 481.547}, "read_interval": {"avg_ms": 411.876, "count": 73, "max_ms": 539.997, "p95_ms": 490.428}, "triton_wait": {"avg_ms": 82.82, "count": 73, "max_ms": 110.16, "p95_ms": 104.211}, "visualize": {"avg_ms": 18.314, "count": 73, "max_ms": 26.055, "p95_ms": 23.001}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.957, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 77, "tracking_summary": 3067}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5656.965, "limit_mb": 8796093022207.996}, "detail": {"pid": 43338, "restart_count": 279, "start_count": 280}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 249, "h264_decode_error_count": 433, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:10:26+09:00", "last_error_summary": "[h264 @ 0x55afabbe0b40] error while decoding MB 25 41, bytestream -11", "last_exit_at": "2026-07-02T06:12:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T06:12:43+09:00", "pid": 43338, "process": {"exists": true, "fd_count": 5, "hwm_mb": 142.004, "name": "ffmpeg", "pid": 43338, "rss_mb": 142.004, "state": "S (sleeping)", "threads": 106}, "restart_count": 279, "restart_count_per_hour": 30, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 280}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 752.312, "sample_count": 546, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 368130, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:12:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.798, "last_frame_pushed_at": "2026-07-02T06:12:41+09:00", "last_frame_read_age_sec": 5.16, "last_frame_read_at": "2026-07-02T06:12:40+09:00", "mjpeg": {"age_sec": 4.779, "exists": true, "mtime": "2026-07-02T06:12:41.199828+09:00", "mtime_epoch": 1782940361.1998284, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194569}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:12:45+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 2584.477, "name": "celery", "pid": 237, "rss_mb": 1918.656, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 63, "pushed_frames": 63, "read_frames": 62, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.111, "pushed": 2.111, "read": 2.078}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 22.433, "count": 62, "max_ms": 33.899, "p95_ms": 27.092}, "gstreamer_push": {"avg_ms": 0.874, "count": 63, "max_ms": 1.024, "p95_ms": 0.923}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.281, "count": 62, "max_ms": 70.887, "p95_ms": 65.346}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 83.879, "count": 62, "max_ms": 110.148, "p95_ms": 104.199}, "yolo11_640_fire_11class": {"avg_ms": 53.94, "count": 62, "max_ms": 74.239, "p95_ms": 68.816}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.491, "count": 62, "max_ms": 106.455, "p95_ms": 90.671}}, "truncated_count": 0}, "processing_total": {"avg_ms": 394.204, "count": 63, "max_ms": 531.119, "p95_ms": 469.772}, "read_interval": {"avg_ms": 404.109, "count": 62, "max_ms": 539.997, "p95_ms": 478.948}, "triton_wait": {"avg_ms": 84.008, "count": 62, "max_ms": 110.16, "p95_ms": 104.211}, "visualize": {"avg_ms": 17.915, "count": 62, "max_ms": 23.001, "p95_ms": 21.733}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.843, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 83, "tracking_summary": 3070}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5645.176, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55f8f3536800] error while decoding MB 21 24, bytestream -11", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 250, "h264_decode_error_count": 434, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:14:56+09:00", "last_error_summary": "[h264 @ 0x55f8f3536800] error while decoding MB 21 24, bytestream -11", "last_exit_at": "2026-07-02T06:12:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.817, "last_started_at": "2026-07-02T06:12:43+09:00", "pid": 43338, "process": {"exists": false, "pid": 43338}, "restart_count": 279, "restart_count_per_hour": 29, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 280}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 692.226, "sample_count": 608, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 368405, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:14:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.046, "last_frame_pushed_at": "2026-07-02T06:14:56+09:00", "last_frame_read_age_sec": 0.038, "last_frame_read_at": "2026-07-02T06:14:56+09:00", "mjpeg": {"age_sec": 0.028, "exists": true, "mtime": "2026-07-02T06:14:56.766878+09:00", "mtime_epoch": 1782940496.7668784, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198790}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:14:56+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 2584.477, "name": "celery", "pid": 237, "rss_mb": 1854.492, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 12.57, "recorders": [{"age_sec": 12.57, "frame_count": 76, "is_recording": true, "is_stopping": false, "pid": 43445, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.891, "name": "ffmpeg", "pid": 43445, "rss_mb": 186.891, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 24, "returncode": null, "running": true, "started_at": "2026-07-02T06:14:44+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_061444_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [43445], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 64, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.151, "pushed": 2.151, "read": 2.151}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.612, "count": 63, "max_ms": 33.891, "p95_ms": 31.331}, "gstreamer_push": {"avg_ms": 0.901, "count": 64, "max_ms": 1.518, "p95_ms": 1.061}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.479, "count": 63, "max_ms": 67.48, "p95_ms": 61.83}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.555, "count": 63, "max_ms": 106.542, "p95_ms": 104.779}, "yolo11_640_fire_11class": {"avg_ms": 52.772, "count": 63, "max_ms": 76.625, "p95_ms": 69.892}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.059, "count": 63, "max_ms": 105.101, "p95_ms": 96.514}}, "truncated_count": 0}, "processing_total": {"avg_ms": 460.47, "count": 64, "max_ms": 1113.141, "p95_ms": 523.44}, "read_interval": {"avg_ms": 470.121, "count": 64, "max_ms": 1123.206, "p95_ms": 533.063}, "triton_wait": {"avg_ms": 81.569, "count": 63, "max_ms": 106.554, "p95_ms": 104.793}, "visualize": {"avg_ms": 16.954, "count": 63, "max_ms": 22.613, "p95_ms": 19.957}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.752, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 83, "tracking_summary": 3070}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5643.684, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T06:14:56+09:00", "last_error_summary": "[h264 @ 0x55f8f3536800] error while decoding MB 21 24, bytestream -11", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 250, "h264_decode_error_count": 434, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:14:56+09:00", "last_error_summary": "[h264 @ 0x55f8f3536800] error while decoding MB 21 24, bytestream -11", "last_exit_at": "2026-07-02T06:14:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 133.224, "last_started_at": "2026-07-02T06:12:43+09:00", "pid": 43338, "process": {"exists": false, "pid": 43338}, "restart_count": 279, "restart_count_per_hour": 29, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 280}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 689.954, "sample_count": 608, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 368406, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:14:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T06:14:57+09:00", "last_frame_read_age_sec": 0.446, "last_frame_read_at": "2026-07-02T06:14:56+09:00", "mjpeg": {"age_sec": 0.435, "exists": true, "mtime": "2026-07-02T06:14:56.766878+09:00", "mtime_epoch": 1782940496.7668784, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198790}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:14:57+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2584.477, "name": "celery", "pid": 237, "rss_mb": 1854.492, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 12.977, "recorders": [{"age_sec": 12.977, "frame_count": 77, "is_recording": true, "is_stopping": false, "pid": 43445, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.891, "name": "ffmpeg", "pid": 43445, "rss_mb": 186.891, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 23, "returncode": null, "running": true, "started_at": "2026-07-02T06:14:44+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_061444_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [43445], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 63, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.15, "pushed": 2.15, "read": 2.116}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.559, "count": 63, "max_ms": 33.891, "p95_ms": 31.331}, "gstreamer_push": {"avg_ms": 0.9, "count": 64, "max_ms": 1.518, "p95_ms": 1.061}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.744, "count": 63, "max_ms": 67.48, "p95_ms": 61.83}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.599, "count": 63, "max_ms": 106.542, "p95_ms": 104.779}, "yolo11_640_fire_11class": {"avg_ms": 52.881, "count": 63, "max_ms": 76.625, "p95_ms": 69.892}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.199, "count": 63, "max_ms": 105.101, "p95_ms": 96.514}}, "truncated_count": 0}, "processing_total": {"avg_ms": 461.496, "count": 64, "max_ms": 1113.141, "p95_ms": 523.44}, "read_interval": {"avg_ms": 471.495, "count": 63, "max_ms": 1123.206, "p95_ms": 533.063}, "triton_wait": {"avg_ms": 81.612, "count": 63, "max_ms": 106.554, "p95_ms": 104.793}, "visualize": {"avg_ms": 16.898, "count": 63, "max_ms": 22.613, "p95_ms": 19.957}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.773, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 83, "tracking_summary": 3070}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5643.801, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 250, "h264_decode_error_count": 434, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:14:56+09:00", "last_error_summary": "[h264 @ 0x55f8f3536800] error while decoding MB 21 24, bytestream -11", "last_exit_at": "2026-07-02T06:14:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 133.227, "last_started_at": "2026-07-02T06:12:43+09:00", "pid": 43338, "process": {"exists": false, "pid": 43338}, "restart_count": 279, "restart_count_per_hour": 29, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 280}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 689.954, "sample_count": 608, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 368406, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:14:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T06:14:57+09:00", "last_frame_read_age_sec": 0.449, "last_frame_read_at": "2026-07-02T06:14:56+09:00", "mjpeg": {"age_sec": 0.438, "exists": true, "mtime": "2026-07-02T06:14:56.766878+09:00", "mtime_epoch": 1782940496.7668784, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198790}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:14:57+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2584.477, "name": "celery", "pid": 237, "rss_mb": 1854.492, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 12.98, "recorders": [{"age_sec": 12.98, "frame_count": 77, "is_recording": true, "is_stopping": false, "pid": 43445, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.891, "name": "ffmpeg", "pid": 43445, "rss_mb": 186.891, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 23, "returncode": null, "running": true, "started_at": "2026-07-02T06:14:44+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_061444_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [43445], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 63, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.149, "pushed": 2.149, "read": 2.116}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.559, "count": 63, "max_ms": 33.891, "p95_ms": 31.331}, "gstreamer_push": {"avg_ms": 0.9, "count": 64, "max_ms": 1.518, "p95_ms": 1.061}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.744, "count": 63, "max_ms": 67.48, "p95_ms": 61.83}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.599, "count": 63, "max_ms": 106.542, "p95_ms": 104.779}, "yolo11_640_fire_11class": {"avg_ms": 52.881, "count": 63, "max_ms": 76.625, "p95_ms": 69.892}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.199, "count": 63, "max_ms": 105.101, "p95_ms": 96.514}}, "truncated_count": 0}, "processing_total": {"avg_ms": 461.496, "count": 64, "max_ms": 1113.141, "p95_ms": 523.44}, "read_interval": {"avg_ms": 471.495, "count": 63, "max_ms": 1123.206, "p95_ms": 533.063}, "triton_wait": {"avg_ms": 81.612, "count": 63, "max_ms": 106.554, "p95_ms": 104.793}, "visualize": {"avg_ms": 16.898, "count": 63, "max_ms": 22.613, "p95_ms": 19.957}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.776, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 83, "tracking_summary": 3070}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5752.898, "limit_mb": 8796093022207.996}, "detail": {"pid": 43505, "restart_count": 280, "start_count": 281}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 250, "h264_decode_error_count": 434, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:14:56+09:00", "last_error_summary": "[h264 @ 0x55f8f3536800] error while decoding MB 21 24, bytestream -11", "last_exit_at": "2026-07-02T06:14:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T06:14:59+09:00", "pid": 43505, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.117, "name": "ffmpeg", "pid": 43505, "rss_mb": 139.117, "state": "S (sleeping)", "threads": 106}, "restart_count": 280, "restart_count_per_hour": 30, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 281}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 677.026, "sample_count": 604, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 368406, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:14:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.086, "last_frame_pushed_at": "2026-07-02T06:14:57+09:00", "last_frame_read_age_sec": 4.527, "last_frame_read_at": "2026-07-02T06:14:56+09:00", "mjpeg": {"age_sec": 4.069, "exists": true, "mtime": "2026-07-02T06:14:57.214888+09:00", "mtime_epoch": 1782940497.2148883, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199007}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:15:01+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 2584.477, "name": "celery", "pid": 237, "rss_mb": 1854.492, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 17.059, "recorders": [{"age_sec": 17.059, "frame_count": 77, "is_recording": true, "is_stopping": false, "pid": 43445, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.891, "name": "ffmpeg", "pid": 43445, "rss_mb": 186.891, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 23, "returncode": null, "running": true, "started_at": "2026-07-02T06:14:44+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_061444_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [43445], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 54, "pushed_frames": 54, "read_frames": 53, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.804, "pushed": 1.804, "read": 1.77}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 26.387, "count": 53, "max_ms": 33.891, "p95_ms": 32.173}, "gstreamer_push": {"avg_ms": 0.907, "count": 54, "max_ms": 1.518, "p95_ms": 1.255}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.258, "count": 53, "max_ms": 67.48, "p95_ms": 62.343}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.19, "count": 53, "max_ms": 106.542, "p95_ms": 105.039}, "yolo11_640_fire_11class": {"avg_ms": 53.638, "count": 53, "max_ms": 76.625, "p95_ms": 71.164}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.786, "count": 53, "max_ms": 105.101, "p95_ms": 98.106}}, "truncated_count": 0}, "processing_total": {"avg_ms": 477.324, "count": 54, "max_ms": 1113.141, "p95_ms": 546.945}, "read_interval": {"avg_ms": 487.734, "count": 53, "max_ms": 1123.206, "p95_ms": 555.809}, "triton_wait": {"avg_ms": 82.204, "count": 53, "max_ms": 106.554, "p95_ms": 105.051}, "visualize": {"avg_ms": 17.283, "count": 53, "max_ms": 22.613, "p95_ms": 20.159}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.94, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 87, "tracking_summary": 3072}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5340.191, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x559e663abc00] error while decoding MB 8 16, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 251, "h264_decode_error_count": 435, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:17:22+09:00", "last_error_summary": "[h264 @ 0x559e663abc00] error while decoding MB 8 16, bytestream -7", "last_exit_at": "2026-07-02T06:14:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 143.303, "last_started_at": "2026-07-02T06:14:59+09:00", "pid": 43505, "process": {"exists": false, "pid": 43505}, "restart_count": 280, "restart_count_per_hour": 29, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 281}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 587.354, "sample_count": 587, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 368670, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:17:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-02T06:17:22+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-02T06:17:22+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-02T06:17:22.541065+09:00", "mtime_epoch": 1782940642.5410652, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205092}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:17:22+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 2584.477, "name": "celery", "pid": 237, "rss_mb": 1656.855, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 10.206, "recorders": [{"age_sec": 10.206, "frame_count": 66, "is_recording": true, "is_stopping": false, "pid": 43612, "process": {"exists": true, "fd_count": 5, "hwm_mb": 187.34, "name": "ffmpeg", "pid": 43612, "rss_mb": 187.34, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 34, "returncode": null, "running": true, "started_at": "2026-07-02T06:17:12+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_061712_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [43612], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 50, "pushed_frames": 50, "read_frames": 50, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.697, "pushed": 1.697, "read": 1.697}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 35.753, "count": 50, "max_ms": 45.964, "p95_ms": 42.641}, "gstreamer_push": {"avg_ms": 0.874, "count": 50, "max_ms": 0.941, "p95_ms": 0.908}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.366, "count": 50, "max_ms": 66.121, "p95_ms": 61.452}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 85.356, "count": 50, "max_ms": 105.645, "p95_ms": 102.677}, "yolo11_640_fire_11class": {"avg_ms": 55.631, "count": 50, "max_ms": 71.974, "p95_ms": 71.203}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.142, "count": 50, "max_ms": 99.662, "p95_ms": 94.935}}, "truncated_count": 0}, "processing_total": {"avg_ms": 590.038, "count": 50, "max_ms": 1240.05, "p95_ms": 617.789}, "read_interval": {"avg_ms": 599.913, "count": 50, "max_ms": 1249.682, "p95_ms": 629.89}, "triton_wait": {"avg_ms": 85.496, "count": 50, "max_ms": 105.658, "p95_ms": 102.69}, "visualize": {"avg_ms": 20.344, "count": 50, "max_ms": 26.502, "p95_ms": 25.276}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.467, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 87, "tracking_summary": 3072}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5338.602, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T06:17:22+09:00", "last_error_summary": "[h264 @ 0x559e663abc00] error while decoding MB 8 16, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 251, "h264_decode_error_count": 435, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:17:22+09:00", "last_error_summary": "[h264 @ 0x559e663abc00] error while decoding MB 8 16, bytestream -7", "last_exit_at": "2026-07-02T06:17:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 143.878, "last_started_at": "2026-07-02T06:14:59+09:00", "pid": 43505, "process": {"exists": false, "pid": 43505}, "restart_count": 280, "restart_count_per_hour": 29, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 281}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 590.232, "sample_count": 586, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 368671, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:17:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T06:17:23+09:00", "last_frame_read_age_sec": 0.625, "last_frame_read_at": "2026-07-02T06:17:22+09:00", "mjpeg": {"age_sec": 0.619, "exists": true, "mtime": "2026-07-02T06:17:22.541065+09:00", "mtime_epoch": 1782940642.5410652, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205092}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:17:23+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2584.477, "name": "celery", "pid": 237, "rss_mb": 1656.855, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 10.78, "recorders": [{"age_sec": 10.78, "frame_count": 67, "is_recording": true, "is_stopping": false, "pid": 43612, "process": {"exists": true, "fd_count": 5, "hwm_mb": 187.34, "name": "ffmpeg", "pid": 43612, "rss_mb": 187.34, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 33, "returncode": null, "running": true, "started_at": "2026-07-02T06:17:12+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_061712_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [43612], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 50, "pushed_frames": 50, "read_frames": 49, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.697, "pushed": 1.697, "read": 1.663}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 35.673, "count": 50, "max_ms": 45.964, "p95_ms": 41.421}, "gstreamer_push": {"avg_ms": 0.874, "count": 50, "max_ms": 0.941, "p95_ms": 0.908}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.566, "count": 50, "max_ms": 66.121, "p95_ms": 61.452}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 85.444, "count": 50, "max_ms": 105.645, "p95_ms": 102.677}, "yolo11_640_fire_11class": {"avg_ms": 55.834, "count": 50, "max_ms": 71.974, "p95_ms": 71.203}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.355, "count": 50, "max_ms": 99.662, "p95_ms": 94.935}}, "truncated_count": 0}, "processing_total": {"avg_ms": 590.848, "count": 50, "max_ms": 1240.05, "p95_ms": 619.926}, "read_interval": {"avg_ms": 600.147, "count": 49, "max_ms": 1249.682, "p95_ms": 629.89}, "triton_wait": {"avg_ms": 85.584, "count": 50, "max_ms": 105.658, "p95_ms": 102.69}, "visualize": {"avg_ms": 20.277, "count": 50, "max_ms": 26.502, "p95_ms": 24.409}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.457, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 87, "tracking_summary": 3072}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5338.594, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 251, "h264_decode_error_count": 435, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:17:22+09:00", "last_error_summary": "[h264 @ 0x559e663abc00] error while decoding MB 8 16, bytestream -7", "last_exit_at": "2026-07-02T06:17:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 143.881, "last_started_at": "2026-07-02T06:14:59+09:00", "pid": 43505, "process": {"exists": false, "pid": 43505}, "restart_count": 280, "restart_count_per_hour": 29, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 281}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 590.232, "sample_count": 586, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 368671, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:17:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T06:17:23+09:00", "last_frame_read_age_sec": 0.628, "last_frame_read_at": "2026-07-02T06:17:22+09:00", "mjpeg": {"age_sec": 0.622, "exists": true, "mtime": "2026-07-02T06:17:22.541065+09:00", "mtime_epoch": 1782940642.5410652, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205092}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:17:23+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2584.477, "name": "celery", "pid": 237, "rss_mb": 1656.855, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 10.783, "recorders": [{"age_sec": 10.783, "frame_count": 67, "is_recording": true, "is_stopping": false, "pid": 43612, "process": {"exists": true, "fd_count": 5, "hwm_mb": 187.34, "name": "ffmpeg", "pid": 43612, "rss_mb": 187.34, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 33, "returncode": null, "running": true, "started_at": "2026-07-02T06:17:12+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_061712_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [43612], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 50, "pushed_frames": 50, "read_frames": 49, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.697, "pushed": 1.697, "read": 1.663}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 35.673, "count": 50, "max_ms": 45.964, "p95_ms": 41.421}, "gstreamer_push": {"avg_ms": 0.874, "count": 50, "max_ms": 0.941, "p95_ms": 0.908}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.566, "count": 50, "max_ms": 66.121, "p95_ms": 61.452}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 85.444, "count": 50, "max_ms": 105.645, "p95_ms": 102.677}, "yolo11_640_fire_11class": {"avg_ms": 55.834, "count": 50, "max_ms": 71.974, "p95_ms": 71.203}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.355, "count": 50, "max_ms": 99.662, "p95_ms": 94.935}}, "truncated_count": 0}, "processing_total": {"avg_ms": 590.848, "count": 50, "max_ms": 1240.05, "p95_ms": 619.926}, "read_interval": {"avg_ms": 600.147, "count": 49, "max_ms": 1249.682, "p95_ms": 629.89}, "triton_wait": {"avg_ms": 85.584, "count": 50, "max_ms": 105.658, "p95_ms": 102.69}, "visualize": {"avg_ms": 20.277, "count": 50, "max_ms": 26.502, "p95_ms": 24.409}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.46, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 87, "tracking_summary": 3072}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5442.348, "limit_mb": 8796093022207.996}, "detail": {"pid": 43672, "restart_count": 281, "start_count": 282}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 251, "h264_decode_error_count": 435, "h264_decode_error_count_per_hour": 29, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:17:22+09:00", "last_error_summary": "[h264 @ 0x559e663abc00] error while decoding MB 8 16, bytestream -7", "last_exit_at": "2026-07-02T06:17:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T06:17:25+09:00", "pid": 43672, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.84, "name": "ffmpeg", "pid": 43672, "rss_mb": 138.84, "state": "S (sleeping)", "threads": 106}, "restart_count": 281, "restart_count_per_hour": 30, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 282}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 590.232, "sample_count": 577, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 368671, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:17:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.21, "last_frame_pushed_at": "2026-07-02T06:17:23+09:00", "last_frame_read_age_sec": 4.831, "last_frame_read_at": "2026-07-02T06:17:22+09:00", "mjpeg": {"age_sec": 4.193, "exists": true, "mtime": "2026-07-02T06:17:23.173052+09:00", "mtime_epoch": 1782940643.1730523, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205507}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:17:27+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 2584.477, "name": "celery", "pid": 237, "rss_mb": 1656.855, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 14.987, "recorders": [{"age_sec": 14.987, "frame_count": 67, "is_recording": true, "is_stopping": false, "pid": 43612, "process": {"exists": true, "fd_count": 5, "hwm_mb": 187.34, "name": "ffmpeg", "pid": 43612, "rss_mb": 187.34, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 33, "returncode": null, "running": true, "started_at": "2026-07-02T06:17:12+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_061712_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [43612], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 43, "pushed_frames": 43, "read_frames": 42, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.444, "pushed": 1.444, "read": 1.41}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 35.912, "count": 42, "max_ms": 45.964, "p95_ms": 41.421}, "gstreamer_push": {"avg_ms": 0.872, "count": 43, "max_ms": 0.941, "p95_ms": 0.905}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.584, "count": 42, "max_ms": 66.121, "p95_ms": 61.094}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 83.947, "count": 42, "max_ms": 105.645, "p95_ms": 100.931}, "yolo11_640_fire_11class": {"avg_ms": 54.342, "count": 42, "max_ms": 71.974, "p95_ms": 69.375}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.218, "count": 42, "max_ms": 95.334, "p95_ms": 94.202}}, "truncated_count": 0}, "processing_total": {"avg_ms": 598.473, "count": 43, "max_ms": 1240.05, "p95_ms": 619.926}, "read_interval": {"avg_ms": 607.754, "count": 42, "max_ms": 1249.682, "p95_ms": 629.89}, "triton_wait": {"avg_ms": 84.111, "count": 42, "max_ms": 105.658, "p95_ms": 100.944}, "visualize": {"avg_ms": 19.938, "count": 42, "max_ms": 24.409, "p95_ms": 23.758}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.788, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 90, "tracking_summary": 3074}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5340.895, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x562d73894000] error while decoding MB 10 24, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 252, "h264_decode_error_count": 436, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:19:41+09:00", "last_error_summary": "[h264 @ 0x562d73894000] error while decoding MB 10 24, bytestream -9", "last_exit_at": "2026-07-02T06:17:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 136.152, "last_started_at": "2026-07-02T06:17:25+09:00", "pid": 43672, "process": {"exists": false, "pid": 43672}, "restart_count": 281, "restart_count_per_hour": 29, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 282}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 594.463, "sample_count": 571, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 368945, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:19:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-02T06:19:41+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-02T06:19:41+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-02T06:19:41.478891+09:00", "mtime_epoch": 1782940781.4788911, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203570}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:19:41+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 2584.477, "name": "celery", "pid": 237, "rss_mb": 1656.855, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 6.721, "recorders": [{"age_sec": 6.721, "frame_count": 61, "is_recording": true, "is_stopping": false, "pid": 43779, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.961, "name": "ffmpeg", "pid": 43779, "rss_mb": 186.961, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 39, "returncode": null, "running": true, "started_at": "2026-07-02T06:19:34+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_061934_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [43779], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 61, "pushed_frames": 61, "read_frames": 61, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.034, "pushed": 2.034, "read": 2.034}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 29.441, "count": 60, "max_ms": 39.519, "p95_ms": 37.684}, "gstreamer_push": {"avg_ms": 0.87, "count": 61, "max_ms": 1.296, "p95_ms": 0.9}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.005, "count": 60, "max_ms": 70.044, "p95_ms": 63.759}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.202, "count": 60, "max_ms": 97.181, "p95_ms": 94.748}, "yolo11_640_fire_11class": {"avg_ms": 54.373, "count": 60, "max_ms": 75.831, "p95_ms": 71.211}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.14, "count": 60, "max_ms": 91.647, "p95_ms": 90.534}}, "truncated_count": 0}, "processing_total": {"avg_ms": 488.985, "count": 61, "max_ms": 1177.049, "p95_ms": 565.745}, "read_interval": {"avg_ms": 498.646, "count": 61, "max_ms": 1186.982, "p95_ms": 575.347}, "triton_wait": {"avg_ms": 73.081, "count": 60, "max_ms": 97.193, "p95_ms": 94.759}, "visualize": {"avg_ms": 17.0, "count": 60, "max_ms": 21.924, "p95_ms": 20.919}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 90, "tracking_summary": 3074}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5339.77, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T06:19:41+09:00", "last_error_summary": "[h264 @ 0x562d73894000] error while decoding MB 10 24, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 252, "h264_decode_error_count": 436, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:19:41+09:00", "last_error_summary": "[h264 @ 0x562d73894000] error while decoding MB 10 24, bytestream -9", "last_exit_at": "2026-07-02T06:19:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 136.632, "last_started_at": "2026-07-02T06:17:25+09:00", "pid": 43672, "process": {"exists": false, "pid": 43672}, "restart_count": 281, "restart_count_per_hour": 29, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 282}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 593.678, "sample_count": 572, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 368946, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:19:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T06:19:41+09:00", "last_frame_read_age_sec": 0.53, "last_frame_read_at": "2026-07-02T06:19:41+09:00", "mjpeg": {"age_sec": 0.518, "exists": true, "mtime": "2026-07-02T06:19:41.478891+09:00", "mtime_epoch": 1782940781.4788911, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203570}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:19:41+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2584.477, "name": "celery", "pid": 237, "rss_mb": 1656.855, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 7.199, "recorders": [{"age_sec": 7.199, "frame_count": 62, "is_recording": true, "is_stopping": false, "pid": 43779, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.961, "name": "ffmpeg", "pid": 43779, "rss_mb": 186.961, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 38, "returncode": null, "running": true, "started_at": "2026-07-02T06:19:34+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_061934_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [43779], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 61, "pushed_frames": 61, "read_frames": 60, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.037, "pushed": 2.037, "read": 2.003}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 29.613, "count": 60, "max_ms": 39.519, "p95_ms": 37.684}, "gstreamer_push": {"avg_ms": 0.87, "count": 61, "max_ms": 1.296, "p95_ms": 0.9}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.373, "count": 60, "max_ms": 70.044, "p95_ms": 63.759}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.866, "count": 60, "max_ms": 97.181, "p95_ms": 94.748}, "yolo11_640_fire_11class": {"avg_ms": 54.132, "count": 60, "max_ms": 75.831, "p95_ms": 70.032}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.787, "count": 60, "max_ms": 91.647, "p95_ms": 90.534}}, "truncated_count": 0}, "processing_total": {"avg_ms": 489.82, "count": 61, "max_ms": 1177.049, "p95_ms": 565.745}, "read_interval": {"avg_ms": 498.891, "count": 60, "max_ms": 1186.982, "p95_ms": 575.347}, "triton_wait": {"avg_ms": 72.746, "count": 60, "max_ms": 97.193, "p95_ms": 94.759}, "visualize": {"avg_ms": 17.028, "count": 60, "max_ms": 21.924, "p95_ms": 20.919}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.952, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 90, "tracking_summary": 3074}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5339.809, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 252, "h264_decode_error_count": 436, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:19:41+09:00", "last_error_summary": "[h264 @ 0x562d73894000] error while decoding MB 10 24, bytestream -9", "last_exit_at": "2026-07-02T06:19:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 136.635, "last_started_at": "2026-07-02T06:17:25+09:00", "pid": 43672, "process": {"exists": false, "pid": 43672}, "restart_count": 281, "restart_count_per_hour": 29, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 282}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 593.678, "sample_count": 572, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 368946, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:19:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T06:19:41+09:00", "last_frame_read_age_sec": 0.533, "last_frame_read_at": "2026-07-02T06:19:41+09:00", "mjpeg": {"age_sec": 0.522, "exists": true, "mtime": "2026-07-02T06:19:41.478891+09:00", "mtime_epoch": 1782940781.4788911, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203570}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:19:42+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2584.477, "name": "celery", "pid": 237, "rss_mb": 1656.855, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 7.202, "recorders": [{"age_sec": 7.202, "frame_count": 62, "is_recording": true, "is_stopping": false, "pid": 43779, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.961, "name": "ffmpeg", "pid": 43779, "rss_mb": 186.961, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 38, "returncode": null, "running": true, "started_at": "2026-07-02T06:19:34+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_061934_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [43779], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 61, "pushed_frames": 61, "read_frames": 60, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.036, "pushed": 2.036, "read": 2.003}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 29.613, "count": 60, "max_ms": 39.519, "p95_ms": 37.684}, "gstreamer_push": {"avg_ms": 0.87, "count": 61, "max_ms": 1.296, "p95_ms": 0.9}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.373, "count": 60, "max_ms": 70.044, "p95_ms": 63.759}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.866, "count": 60, "max_ms": 97.181, "p95_ms": 94.748}, "yolo11_640_fire_11class": {"avg_ms": 54.132, "count": 60, "max_ms": 75.831, "p95_ms": 70.032}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.787, "count": 60, "max_ms": 91.647, "p95_ms": 90.534}}, "truncated_count": 0}, "processing_total": {"avg_ms": 489.82, "count": 61, "max_ms": 1177.049, "p95_ms": 565.745}, "read_interval": {"avg_ms": 498.891, "count": 60, "max_ms": 1186.982, "p95_ms": 575.347}, "triton_wait": {"avg_ms": 72.746, "count": 60, "max_ms": 97.193, "p95_ms": 94.759}, "visualize": {"avg_ms": 17.028, "count": 60, "max_ms": 21.924, "p95_ms": 20.919}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.955, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 90, "tracking_summary": 3074}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5444.824, "limit_mb": 8796093022207.996}, "detail": {"pid": 43839, "restart_count": 282, "start_count": 283}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 252, "h264_decode_error_count": 436, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:19:41+09:00", "last_error_summary": "[h264 @ 0x562d73894000] error while decoding MB 10 24, bytestream -9", "last_exit_at": "2026-07-02T06:19:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T06:19:44+09:00", "pid": 43839, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.379, "name": "ffmpeg", "pid": 43839, "rss_mb": 139.379, "state": "S (sleeping)", "threads": 106}, "restart_count": 282, "restart_count_per_hour": 29, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 283}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 593.678, "sample_count": 564, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 368946, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:19:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.541, "last_frame_pushed_at": "2026-07-02T06:19:41+09:00", "last_frame_read_age_sec": 5.068, "last_frame_read_at": "2026-07-02T06:19:41+09:00", "mjpeg": {"age_sec": 4.525, "exists": true, "mtime": "2026-07-02T06:19:42.010885+09:00", "mtime_epoch": 1782940782.010885, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200914}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:19:46+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 2584.477, "name": "celery", "pid": 237, "rss_mb": 1656.855, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 11.739, "recorders": [{"age_sec": 11.739, "frame_count": 62, "is_recording": true, "is_stopping": false, "pid": 43779, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.961, "name": "ffmpeg", "pid": 43779, "rss_mb": 186.961, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 38, "returncode": null, "running": true, "started_at": "2026-07-02T06:19:34+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_061934_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [43779], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 51, "pushed_frames": 51, "read_frames": 50, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.703, "pushed": 1.703, "read": 1.67}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 30.464, "count": 50, "max_ms": 39.519, "p95_ms": 38.343}, "gstreamer_push": {"avg_ms": 0.864, "count": 51, "max_ms": 0.929, "p95_ms": 0.9}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.662, "count": 50, "max_ms": 70.044, "p95_ms": 63.936}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.264, "count": 50, "max_ms": 97.181, "p95_ms": 95.776}, "yolo11_640_fire_11class": {"avg_ms": 54.433, "count": 50, "max_ms": 75.831, "p95_ms": 72.426}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.063, "count": 50, "max_ms": 91.647, "p95_ms": 90.65}}, "truncated_count": 0}, "processing_total": {"avg_ms": 497.513, "count": 51, "max_ms": 1177.049, "p95_ms": 575.132}, "read_interval": {"avg_ms": 506.537, "count": 50, "max_ms": 1186.982, "p95_ms": 583.747}, "triton_wait": {"avg_ms": 73.074, "count": 50, "max_ms": 97.193, "p95_ms": 95.789}, "visualize": {"avg_ms": 17.115, "count": 50, "max_ms": 21.924, "p95_ms": 20.943}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.942, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 2, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 92, "tracking_summary": 3077}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5352.77, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55a7afd3a880] error while decoding MB 60 40, bytestream -15", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 253, "h264_decode_error_count": 437, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:21:58+09:00", "last_error_summary": "[h264 @ 0x55a7afd3a880] error while decoding MB 60 40, bytestream -15", "last_exit_at": "2026-07-02T06:19:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 134.259, "last_started_at": "2026-07-02T06:19:44+09:00", "pid": 43839, "process": {"exists": false, "pid": 43839}, "restart_count": 282, "restart_count_per_hour": 28, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 283}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 583.522, "sample_count": 644, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 369274, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:21:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-02T06:21:58+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-02T06:21:58+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-02T06:21:58.749801+09:00", "mtime_epoch": 1782940918.7498014, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204592}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:21:58+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 2584.477, "name": "celery", "pid": 237, "rss_mb": 1657.113, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 17.662, "recorders": [{"age_sec": 17.662, "frame_count": 84, "is_recording": true, "is_stopping": false, "pid": 43946, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.801, "name": "ffmpeg", "pid": 43946, "rss_mb": 186.801, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 16, "returncode": null, "running": true, "started_at": "2026-07-02T06:21:41+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_062140_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [43946], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 64, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.139, "pushed": 2.139, "read": 2.139}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 26.961, "count": 63, "max_ms": 38.747, "p95_ms": 33.643}, "gstreamer_push": {"avg_ms": 0.873, "count": 64, "max_ms": 0.992, "p95_ms": 0.927}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.961, "count": 63, "max_ms": 74.043, "p95_ms": 68.948}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.278, "count": 63, "max_ms": 100.24, "p95_ms": 98.764}, "yolo11_640_fire_11class": {"avg_ms": 52.978, "count": 63, "max_ms": 77.758, "p95_ms": 69.629}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.88, "count": 63, "max_ms": 98.98, "p95_ms": 95.461}}, "truncated_count": 0}, "processing_total": {"avg_ms": 463.203, "count": 64, "max_ms": 1057.195, "p95_ms": 563.206}, "read_interval": {"avg_ms": 472.801, "count": 64, "max_ms": 1067.188, "p95_ms": 575.176}, "triton_wait": {"avg_ms": 77.631, "count": 63, "max_ms": 100.252, "p95_ms": 98.992}, "visualize": {"avg_ms": 18.49, "count": 63, "max_ms": 27.74, "p95_ms": 22.73}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.918, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 2, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 92, "tracking_summary": 3077}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5351.32, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T06:21:58+09:00", "last_error_summary": "[h264 @ 0x55a7afd3a880] error while decoding MB 60 40, bytestream -15", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 253, "h264_decode_error_count": 437, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:21:58+09:00", "last_error_summary": "[h264 @ 0x55a7afd3a880] error while decoding MB 60 40, bytestream -15", "last_exit_at": "2026-07-02T06:21:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 134.744, "last_started_at": "2026-07-02T06:19:44+09:00", "pid": 43839, "process": {"exists": false, "pid": 43839}, "restart_count": 282, "restart_count_per_hour": 28, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 283}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 580.639, "sample_count": 644, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 369275, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:21:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T06:21:59+09:00", "last_frame_read_age_sec": 0.535, "last_frame_read_at": "2026-07-02T06:21:58+09:00", "mjpeg": {"age_sec": 0.529, "exists": true, "mtime": "2026-07-02T06:21:58.749801+09:00", "mtime_epoch": 1782940918.7498014, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204592}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:21:59+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2584.477, "name": "celery", "pid": 237, "rss_mb": 1657.113, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 18.146, "recorders": [{"age_sec": 18.146, "frame_count": 85, "is_recording": true, "is_stopping": false, "pid": 43946, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.801, "name": "ffmpeg", "pid": 43946, "rss_mb": 186.801, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 15, "returncode": null, "running": true, "started_at": "2026-07-02T06:21:41+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_062140_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [43946], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 63, "pushed_frames": 63, "read_frames": 62, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.127, "pushed": 2.127, "read": 2.093}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 27.108, "count": 63, "max_ms": 38.747, "p95_ms": 33.643}, "gstreamer_push": {"avg_ms": 0.874, "count": 63, "max_ms": 0.992, "p95_ms": 0.927}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.293, "count": 63, "max_ms": 74.043, "p95_ms": 69.221}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.617, "count": 63, "max_ms": 100.24, "p95_ms": 98.764}, "yolo11_640_fire_11class": {"avg_ms": 53.308, "count": 63, "max_ms": 77.758, "p95_ms": 69.629}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.132, "count": 63, "max_ms": 98.98, "p95_ms": 95.461}}, "truncated_count": 0}, "processing_total": {"avg_ms": 466.717, "count": 63, "max_ms": 1057.195, "p95_ms": 563.206}, "read_interval": {"avg_ms": 475.308, "count": 62, "max_ms": 1067.188, "p95_ms": 575.176}, "triton_wait": {"avg_ms": 77.97, "count": 63, "max_ms": 100.252, "p95_ms": 98.992}, "visualize": {"avg_ms": 18.562, "count": 63, "max_ms": 27.74, "p95_ms": 22.73}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.62, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 2, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 92, "tracking_summary": 3077}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5351.289, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 253, "h264_decode_error_count": 437, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:21:58+09:00", "last_error_summary": "[h264 @ 0x55a7afd3a880] error while decoding MB 60 40, bytestream -15", "last_exit_at": "2026-07-02T06:21:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 134.747, "last_started_at": "2026-07-02T06:19:44+09:00", "pid": 43839, "process": {"exists": false, "pid": 43839}, "restart_count": 282, "restart_count_per_hour": 28, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 283}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 580.639, "sample_count": 644, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 369275, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:21:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T06:21:59+09:00", "last_frame_read_age_sec": 0.539, "last_frame_read_at": "2026-07-02T06:21:58+09:00", "mjpeg": {"age_sec": 0.532, "exists": true, "mtime": "2026-07-02T06:21:58.749801+09:00", "mtime_epoch": 1782940918.7498014, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204592}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:21:59+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2584.477, "name": "celery", "pid": 237, "rss_mb": 1657.113, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 18.149, "recorders": [{"age_sec": 18.149, "frame_count": 85, "is_recording": true, "is_stopping": false, "pid": 43946, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.801, "name": "ffmpeg", "pid": 43946, "rss_mb": 186.801, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 15, "returncode": null, "running": true, "started_at": "2026-07-02T06:21:41+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_062140_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [43946], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 63, "pushed_frames": 63, "read_frames": 62, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.127, "pushed": 2.127, "read": 2.093}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 27.108, "count": 63, "max_ms": 38.747, "p95_ms": 33.643}, "gstreamer_push": {"avg_ms": 0.874, "count": 63, "max_ms": 0.992, "p95_ms": 0.927}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.293, "count": 63, "max_ms": 74.043, "p95_ms": 69.221}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.617, "count": 63, "max_ms": 100.24, "p95_ms": 98.764}, "yolo11_640_fire_11class": {"avg_ms": 53.308, "count": 63, "max_ms": 77.758, "p95_ms": 69.629}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.132, "count": 63, "max_ms": 98.98, "p95_ms": 95.461}}, "truncated_count": 0}, "processing_total": {"avg_ms": 466.717, "count": 63, "max_ms": 1057.195, "p95_ms": 563.206}, "read_interval": {"avg_ms": 475.308, "count": 62, "max_ms": 1067.188, "p95_ms": 575.176}, "triton_wait": {"avg_ms": 77.97, "count": 63, "max_ms": 100.252, "p95_ms": 98.992}, "visualize": {"avg_ms": 18.562, "count": 63, "max_ms": 27.74, "p95_ms": 22.73}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.624, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 2, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 92, "tracking_summary": 3077}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5460.715, "limit_mb": 8796093022207.996}, "detail": {"pid": 44006, "restart_count": 283, "start_count": 284}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 253, "h264_decode_error_count": 437, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:21:58+09:00", "last_error_summary": "[h264 @ 0x55a7afd3a880] error while decoding MB 60 40, bytestream -15", "last_exit_at": "2026-07-02T06:22:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T06:22:02+09:00", "pid": 44006, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.238, "name": "ffmpeg", "pid": 44006, "rss_mb": 139.238, "state": "S (sleeping)", "threads": 106}, "restart_count": 283, "restart_count_per_hour": 29, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 284}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 575.132, "sample_count": 635, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 369275, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:21:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.864, "last_frame_pushed_at": "2026-07-02T06:21:59+09:00", "last_frame_read_age_sec": 5.396, "last_frame_read_at": "2026-07-02T06:21:58+09:00", "mjpeg": {"age_sec": 4.846, "exists": true, "mtime": "2026-07-02T06:21:59.293799+09:00", "mtime_epoch": 1782940919.2937987, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204643}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:22:04+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 2584.477, "name": "celery", "pid": 237, "rss_mb": 1657.113, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 23.008, "recorders": [{"age_sec": 23.008, "frame_count": 85, "is_recording": true, "is_stopping": false, "pid": 43946, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.801, "name": "ffmpeg", "pid": 43946, "rss_mb": 186.801, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 15, "returncode": null, "running": true, "started_at": "2026-07-02T06:21:41+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_062140_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [43946], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 51, "pushed_frames": 51, "read_frames": 50, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.721, "pushed": 1.721, "read": 1.687}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 27.858, "count": 51, "max_ms": 38.747, "p95_ms": 33.696}, "gstreamer_push": {"avg_ms": 0.878, "count": 51, "max_ms": 0.992, "p95_ms": 0.93}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.093, "count": 51, "max_ms": 74.043, "p95_ms": 70.231}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.527, "count": 51, "max_ms": 100.24, "p95_ms": 98.998}, "yolo11_640_fire_11class": {"avg_ms": 53.576, "count": 51, "max_ms": 77.758, "p95_ms": 70.614}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.32, "count": 51, "max_ms": 98.98, "p95_ms": 96.219}}, "truncated_count": 0}, "processing_total": {"avg_ms": 483.651, "count": 51, "max_ms": 1057.195, "p95_ms": 576.472}, "read_interval": {"avg_ms": 492.439, "count": 50, "max_ms": 1067.188, "p95_ms": 586.522}, "triton_wait": {"avg_ms": 78.939, "count": 51, "max_ms": 100.252, "p95_ms": 99.01}, "visualize": {"avg_ms": 19.194, "count": 51, "max_ms": 27.74, "p95_ms": 23.591}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.637, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 2, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 113, "tracking_summary": 3078}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5824.445, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55be697d7d80] error while decoding MB 2 33, bytestream -38", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 254, "h264_decode_error_count": 438, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:23:56+09:00", "last_error_summary": "[h264 @ 0x55be697d7d80] error while decoding MB 2 33, bytestream -38", "last_exit_at": "2026-07-02T06:22:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 114.065, "last_started_at": "2026-07-02T06:22:02+09:00", "pid": 44006, "process": {"exists": false, "pid": 44006}, "restart_count": 283, "restart_count_per_hour": 28, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 284}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 812.766, "sample_count": 571, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 369421, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:23:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-02T06:23:56+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-02T06:23:56+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-02T06:23:56.157514+09:00", "mtime_epoch": 1782941036.1575143, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 213657}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:23:56+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 2584.477, "name": "celery", "pid": 237, "rss_mb": 2095.559, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 30.722, "recorders": [{"age_sec": 30.722, "frame_count": 88, "is_recording": true, "is_stopping": false, "pid": 44113, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.316, "name": "ffmpeg", "pid": 44113, "rss_mb": 186.316, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 12, "returncode": null, "running": true, "started_at": "2026-07-02T06:23:25+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_062325_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [44113], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 38, "pushed_frames": 38, "read_frames": 38, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.289, "pushed": 1.289, "read": 1.289}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 51.647, "count": 37, "max_ms": 60.097, "p95_ms": 58.995}, "gstreamer_push": {"avg_ms": 0.931, "count": 38, "max_ms": 1.936, "p95_ms": 1.896}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.399, "count": 37, "max_ms": 74.611, "p95_ms": 68.82}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 86.697, "count": 37, "max_ms": 108.992, "p95_ms": 105.764}, "yolo11_640_fire_11class": {"avg_ms": 53.791, "count": 37, "max_ms": 71.124, "p95_ms": 70.044}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.133, "count": 37, "max_ms": 94.577, "p95_ms": 93.137}}, "truncated_count": 0}, "processing_total": {"avg_ms": 802.113, "count": 38, "max_ms": 1428.09, "p95_ms": 1150.862}, "read_interval": {"avg_ms": 811.716, "count": 38, "max_ms": 1437.984, "p95_ms": 1160.882}, "triton_wait": {"avg_ms": 86.71, "count": 37, "max_ms": 109.006, "p95_ms": 105.776}, "visualize": {"avg_ms": 25.125, "count": 37, "max_ms": 35.612, "p95_ms": 31.586}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.472, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 114, "tracking_summary": 3078}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5823.84, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T06:23:56+09:00", "last_error_summary": "[h264 @ 0x55be697d7d80] error while decoding MB 2 33, bytestream -38", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 254, "h264_decode_error_count": 438, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:23:56+09:00", "last_error_summary": "[h264 @ 0x55be697d7d80] error while decoding MB 2 33, bytestream -38", "last_exit_at": "2026-07-02T06:23:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 114.973, "last_started_at": "2026-07-02T06:22:02+09:00", "pid": 44006, "process": {"exists": false, "pid": 44006}, "restart_count": 283, "restart_count_per_hour": 28, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 284}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 813.741, "sample_count": 569, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 369422, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:23:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.002, "last_frame_pushed_at": "2026-07-02T06:23:57+09:00", "last_frame_read_age_sec": 0.963, "last_frame_read_at": "2026-07-02T06:23:56+09:00", "mjpeg": {"age_sec": 0.954, "exists": true, "mtime": "2026-07-02T06:23:56.157514+09:00", "mtime_epoch": 1782941036.1575143, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 213657}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:23:57+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2584.477, "name": "celery", "pid": 237, "rss_mb": 2095.559, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 31.63, "recorders": [{"age_sec": 31.63, "frame_count": 89, "is_recording": true, "is_stopping": false, "pid": 44113, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.316, "name": "ffmpeg", "pid": 44113, "rss_mb": 186.316, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 11, "returncode": null, "running": true, "started_at": "2026-07-02T06:23:25+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_062325_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [44113], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 38, "pushed_frames": 38, "read_frames": 37, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.284, "pushed": 1.284, "read": 1.25}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 51.815, "count": 37, "max_ms": 60.097, "p95_ms": 59.071}, "gstreamer_push": {"avg_ms": 0.932, "count": 38, "max_ms": 1.936, "p95_ms": 1.896}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.186, "count": 37, "max_ms": 74.611, "p95_ms": 68.82}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 86.098, "count": 37, "max_ms": 108.992, "p95_ms": 105.764}, "yolo11_640_fire_11class": {"avg_ms": 53.55, "count": 37, "max_ms": 71.124, "p95_ms": 70.044}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.992, "count": 37, "max_ms": 94.577, "p95_ms": 93.137}}, "truncated_count": 0}, "processing_total": {"avg_ms": 789.782, "count": 38, "max_ms": 1150.862, "p95_ms": 1098.727}, "read_interval": {"avg_ms": 794.79, "count": 37, "max_ms": 1160.882, "p95_ms": 1109.905}, "triton_wait": {"avg_ms": 86.112, "count": 37, "max_ms": 109.006, "p95_ms": 105.776}, "visualize": {"avg_ms": 25.242, "count": 37, "max_ms": 35.612, "p95_ms": 31.586}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.597, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 114, "tracking_summary": 3078}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5823.832, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 254, "h264_decode_error_count": 438, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:23:56+09:00", "last_error_summary": "[h264 @ 0x55be697d7d80] error while decoding MB 2 33, bytestream -38", "last_exit_at": "2026-07-02T06:23:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 114.976, "last_started_at": "2026-07-02T06:22:02+09:00", "pid": 44006, "process": {"exists": false, "pid": 44006}, "restart_count": 283, "restart_count_per_hour": 28, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 284}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 813.741, "sample_count": 569, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 369422, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:23:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T06:23:57+09:00", "last_frame_read_age_sec": 0.966, "last_frame_read_at": "2026-07-02T06:23:56+09:00", "mjpeg": {"age_sec": 0.957, "exists": true, "mtime": "2026-07-02T06:23:56.157514+09:00", "mtime_epoch": 1782941036.1575143, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 213657}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:23:57+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 2584.477, "name": "celery", "pid": 237, "rss_mb": 2095.559, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 31.633, "recorders": [{"age_sec": 31.633, "frame_count": 89, "is_recording": true, "is_stopping": false, "pid": 44113, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.316, "name": "ffmpeg", "pid": 44113, "rss_mb": 186.316, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 11, "returncode": null, "running": true, "started_at": "2026-07-02T06:23:25+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_062325_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [44113], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 38, "pushed_frames": 38, "read_frames": 37, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.284, "pushed": 1.284, "read": 1.25}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 51.815, "count": 37, "max_ms": 60.097, "p95_ms": 59.071}, "gstreamer_push": {"avg_ms": 0.932, "count": 38, "max_ms": 1.936, "p95_ms": 1.896}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.186, "count": 37, "max_ms": 74.611, "p95_ms": 68.82}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 86.098, "count": 37, "max_ms": 108.992, "p95_ms": 105.764}, "yolo11_640_fire_11class": {"avg_ms": 53.55, "count": 37, "max_ms": 71.124, "p95_ms": 70.044}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.992, "count": 37, "max_ms": 94.577, "p95_ms": 93.137}}, "truncated_count": 0}, "processing_total": {"avg_ms": 789.782, "count": 38, "max_ms": 1150.862, "p95_ms": 1098.727}, "read_interval": {"avg_ms": 794.79, "count": 37, "max_ms": 1160.882, "p95_ms": 1109.905}, "triton_wait": {"avg_ms": 86.112, "count": 37, "max_ms": 109.006, "p95_ms": 105.776}, "visualize": {"avg_ms": 25.242, "count": 37, "max_ms": 35.612, "p95_ms": 31.586}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.599, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 114, "tracking_summary": 3078}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5929.621, "limit_mb": 8796093022207.996}, "detail": {"pid": 44173, "restart_count": 284, "start_count": 285}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 254, "h264_decode_error_count": 438, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:23:56+09:00", "last_error_summary": "[h264 @ 0x55be697d7d80] error while decoding MB 2 33, bytestream -38", "last_exit_at": "2026-07-02T06:23:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T06:23:59+09:00", "pid": 44173, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.047, "name": "ffmpeg", "pid": 44173, "rss_mb": 139.047, "state": "S (sleeping)", "threads": 106}, "restart_count": 284, "restart_count_per_hour": 29, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 285}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 813.741, "sample_count": 559, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 369422, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:23:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.518, "last_frame_pushed_at": "2026-07-02T06:23:57+09:00", "last_frame_read_age_sec": 5.479, "last_frame_read_at": "2026-07-02T06:23:56+09:00", "mjpeg": {"age_sec": 4.502, "exists": true, "mtime": "2026-07-02T06:23:57.125514+09:00", "mtime_epoch": 1782941037.1255143, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 214116}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:24:01+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 2584.477, "name": "celery", "pid": 237, "rss_mb": 2095.559, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 36.147, "recorders": [{"age_sec": 36.147, "frame_count": 89, "is_recording": true, "is_stopping": false, "pid": 44113, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.316, "name": "ffmpeg", "pid": 44113, "rss_mb": 186.316, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 11, "returncode": null, "running": true, "started_at": "2026-07-02T06:23:25+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_062325_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [44113], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 33, "pushed_frames": 33, "read_frames": 32, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.114, "pushed": 1.114, "read": 1.08}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 52.155, "count": 32, "max_ms": 60.097, "p95_ms": 59.071}, "gstreamer_push": {"avg_ms": 0.869, "count": 33, "max_ms": 0.917, "p95_ms": 0.904}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.674, "count": 32, "max_ms": 74.611, "p95_ms": 68.82}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 86.435, "count": 32, "max_ms": 108.992, "p95_ms": 105.764}, "yolo11_640_fire_11class": {"avg_ms": 54.047, "count": 32, "max_ms": 71.124, "p95_ms": 70.044}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.742, "count": 32, "max_ms": 94.577, "p95_ms": 93.137}}, "truncated_count": 0}, "processing_total": {"avg_ms": 772.525, "count": 33, "max_ms": 959.502, "p95_ms": 917.577}, "read_interval": {"avg_ms": 776.149, "count": 32, "max_ms": 927.548, "p95_ms": 897.732}, "triton_wait": {"avg_ms": 86.449, "count": 32, "max_ms": 109.006, "p95_ms": 105.776}, "visualize": {"avg_ms": 24.839, "count": 32, "max_ms": 35.612, "p95_ms": 30.61}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.619, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 208, "tracking_summary": 3079}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9595.965, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x564b95244000] error while decoding MB 5 37, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 255, "h264_decode_error_count": 439, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:27:25+09:00", "last_error_summary": "[h264 @ 0x564b95244000] error while decoding MB 5 37, bytestream -9", "last_exit_at": "2026-07-02T06:23:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 205.822, "last_started_at": "2026-07-02T06:23:59+09:00", "pid": 44173, "process": {"exists": false, "pid": 44173}, "restart_count": 284, "restart_count_per_hour": 27, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 285}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 2092.78, "sample_count": 236, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 369538, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:27:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.067, "last_frame_pushed_at": "2026-07-02T06:27:25+09:00", "last_frame_read_age_sec": 0.058, "last_frame_read_at": "2026-07-02T06:27:25+09:00", "mjpeg": {"age_sec": 0.051, "exists": true, "mtime": "2026-07-02T06:27:25.398097+09:00", "mtime_epoch": 1782941245.3980968, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 233668}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:27:25+09:00", "process": {"exists": true, "fd_count": 123, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5693.301, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 33.162, "recorders": [{"age_sec": 33.162, "frame_count": 68, "is_recording": true, "is_stopping": false, "pid": 44341, "process": {"exists": true, "fd_count": 5, "hwm_mb": 187.297, "name": "ffmpeg", "pid": 44341, "rss_mb": 187.297, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 32, "returncode": null, "running": true, "started_at": "2026-07-02T06:26:52+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_062652_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [44341], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 17, "pushed_frames": 17, "read_frames": 17, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.587, "pushed": 0.587, "read": 0.587}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 118.968, "count": 16, "max_ms": 134.957, "p95_ms": 134.957}, "gstreamer_push": {"avg_ms": 0.814, "count": 17, "max_ms": 1.376, "p95_ms": 1.376}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 41.292, "count": 16, "max_ms": 64.483, "p95_ms": 64.483}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.822, "count": 16, "max_ms": 98.489, "p95_ms": 98.489}, "yolo11_640_fire_11class": {"avg_ms": 47.54, "count": 16, "max_ms": 64.971, "p95_ms": 64.971}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 91.375, "count": 16, "max_ms": 117.901, "p95_ms": 117.901}}, "truncated_count": 0}, "processing_total": {"avg_ms": 1804.722, "count": 17, "max_ms": 2323.178, "p95_ms": 2323.178}, "read_interval": {"avg_ms": 1813.449, "count": 17, "max_ms": 2332.392, "p95_ms": 2332.392}, "triton_wait": {"avg_ms": 91.45, "count": 16, "max_ms": 117.914, "p95_ms": 117.914}, "visualize": {"avg_ms": 70.525, "count": 16, "max_ms": 87.198, "p95_ms": 87.198}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 28.98, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 208, "tracking_summary": 3079}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9595.242, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T06:27:25+09:00", "last_error_summary": "[h264 @ 0x564b95244000] error while decoding MB 5 37, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 255, "h264_decode_error_count": 439, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:27:25+09:00", "last_error_summary": "[h264 @ 0x564b95244000] error while decoding MB 5 37, bytestream -9", "last_exit_at": "2026-07-02T06:27:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 207.725, "last_started_at": "2026-07-02T06:23:59+09:00", "pid": 44173, "process": {"exists": false, "pid": 44173}, "restart_count": 284, "restart_count_per_hour": 27, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 285}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 2092.78, "sample_count": 235, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 369539, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:27:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T06:27:27+09:00", "last_frame_read_age_sec": 1.962, "last_frame_read_at": "2026-07-02T06:27:25+09:00", "mjpeg": {"age_sec": 1.954, "exists": true, "mtime": "2026-07-02T06:27:25.398097+09:00", "mtime_epoch": 1782941245.3980968, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 233668}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:27:27+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5693.555, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 35.065, "recorders": [{"age_sec": 35.065, "frame_count": 69, "is_recording": true, "is_stopping": false, "pid": 44341, "process": {"exists": true, "fd_count": 5, "hwm_mb": 187.297, "name": "ffmpeg", "pid": 44341, "rss_mb": 187.297, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 31, "returncode": null, "running": true, "started_at": "2026-07-02T06:26:52+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_062652_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [44341], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 17, "pushed_frames": 17, "read_frames": 16, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.591, "pushed": 0.591, "read": 0.556}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 118.803, "count": 16, "max_ms": 133.212, "p95_ms": 133.212}, "gstreamer_push": {"avg_ms": 0.817, "count": 17, "max_ms": 1.376, "p95_ms": 1.376}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.706, "count": 16, "max_ms": 64.483, "p95_ms": 64.483}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.576, "count": 16, "max_ms": 98.489, "p95_ms": 98.489}, "yolo11_640_fire_11class": {"avg_ms": 47.454, "count": 16, "max_ms": 64.971, "p95_ms": 64.971}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 91.46, "count": 16, "max_ms": 117.901, "p95_ms": 117.901}}, "truncated_count": 0}, "processing_total": {"avg_ms": 1807.637, "count": 17, "max_ms": 2323.178, "p95_ms": 2323.178}, "read_interval": {"avg_ms": 1806.989, "count": 16, "max_ms": 2332.392, "p95_ms": 2332.392}, "triton_wait": {"avg_ms": 91.473, "count": 16, "max_ms": 117.914, "p95_ms": 117.914}, "visualize": {"avg_ms": 71.437, "count": 16, "max_ms": 87.198, "p95_ms": 87.198}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 28.781, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 208, "tracking_summary": 3079}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9595.395, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 255, "h264_decode_error_count": 439, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:27:25+09:00", "last_error_summary": "[h264 @ 0x564b95244000] error while decoding MB 5 37, bytestream -9", "last_exit_at": "2026-07-02T06:27:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 207.728, "last_started_at": "2026-07-02T06:23:59+09:00", "pid": 44173, "process": {"exists": false, "pid": 44173}, "restart_count": 284, "restart_count_per_hour": 27, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 285}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 2092.78, "sample_count": 235, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 369539, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:27:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T06:27:27+09:00", "last_frame_read_age_sec": 1.964, "last_frame_read_at": "2026-07-02T06:27:25+09:00", "mjpeg": {"age_sec": 1.957, "exists": true, "mtime": "2026-07-02T06:27:25.398097+09:00", "mtime_epoch": 1782941245.3980968, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 233668}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:27:27+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5693.555, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 35.067, "recorders": [{"age_sec": 35.067, "frame_count": 69, "is_recording": true, "is_stopping": false, "pid": 44341, "process": {"exists": true, "fd_count": 5, "hwm_mb": 187.297, "name": "ffmpeg", "pid": 44341, "rss_mb": 187.297, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 31, "returncode": null, "running": true, "started_at": "2026-07-02T06:26:52+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_062652_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [44341], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 17, "pushed_frames": 17, "read_frames": 16, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.591, "pushed": 0.591, "read": 0.556}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 118.803, "count": 16, "max_ms": 133.212, "p95_ms": 133.212}, "gstreamer_push": {"avg_ms": 0.817, "count": 17, "max_ms": 1.376, "p95_ms": 1.376}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 40.706, "count": 16, "max_ms": 64.483, "p95_ms": 64.483}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.576, "count": 16, "max_ms": 98.489, "p95_ms": 98.489}, "yolo11_640_fire_11class": {"avg_ms": 47.454, "count": 16, "max_ms": 64.971, "p95_ms": 64.971}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 91.46, "count": 16, "max_ms": 117.901, "p95_ms": 117.901}}, "truncated_count": 0}, "processing_total": {"avg_ms": 1807.637, "count": 17, "max_ms": 2323.178, "p95_ms": 2323.178}, "read_interval": {"avg_ms": 1806.989, "count": 16, "max_ms": 2332.392, "p95_ms": 2332.392}, "triton_wait": {"avg_ms": 91.473, "count": 16, "max_ms": 117.914, "p95_ms": 117.914}, "visualize": {"avg_ms": 71.437, "count": 16, "max_ms": 87.198, "p95_ms": 87.198}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 28.784, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 208, "tracking_summary": 3079}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9707.012, "limit_mb": 8796093022207.996}, "detail": {"pid": 44401, "restart_count": 285, "start_count": 286}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 255, "h264_decode_error_count": 439, "h264_decode_error_count_per_hour": 27, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:27:25+09:00", "last_error_summary": "[h264 @ 0x564b95244000] error while decoding MB 5 37, bytestream -9", "last_exit_at": "2026-07-02T06:27:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T06:27:29+09:00", "pid": 44401, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.27, "name": "ffmpeg", "pid": 44401, "rss_mb": 139.27, "state": "S (sleeping)", "threads": 106}, "restart_count": 285, "restart_count_per_hour": 28, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 286}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 2105.059, "sample_count": 229, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 369539, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:27:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.589, "last_frame_pushed_at": "2026-07-02T06:27:27+09:00", "last_frame_read_age_sec": 6.547, "last_frame_read_at": "2026-07-02T06:27:25+09:00", "mjpeg": {"age_sec": 4.572, "exists": true, "mtime": "2026-07-02T06:27:27.366108+09:00", "mtime_epoch": 1782941247.3661077, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 234240}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:27:31+09:00", "process": {"exists": true, "fd_count": 123, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5693.555, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 39.651, "recorders": [{"age_sec": 39.651, "frame_count": 69, "is_recording": true, "is_stopping": false, "pid": 44341, "process": {"exists": true, "fd_count": 5, "hwm_mb": 187.297, "name": "ffmpeg", "pid": 44341, "rss_mb": 187.297, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 31, "returncode": null, "running": true, "started_at": "2026-07-02T06:26:52+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_062652_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [44341], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 15, "pushed_frames": 15, "read_frames": 14, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.507, "pushed": 0.507, "read": 0.473}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 119.52, "count": 14, "max_ms": 133.212, "p95_ms": 133.212}, "gstreamer_push": {"avg_ms": 0.838, "count": 15, "max_ms": 1.376, "p95_ms": 1.376}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 38.947, "count": 14, "max_ms": 47.95, "p95_ms": 47.95}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.906, "count": 14, "max_ms": 80.642, "p95_ms": 80.642}, "yolo11_640_fire_11class": {"avg_ms": 45.948, "count": 14, "max_ms": 52.493, "p95_ms": 52.493}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 88.685, "count": 14, "max_ms": 105.181, "p95_ms": 105.181}}, "truncated_count": 0}, "processing_total": {"avg_ms": 1798.335, "count": 15, "max_ms": 2323.178, "p95_ms": 2323.178}, "read_interval": {"avg_ms": 1795.716, "count": 14, "max_ms": 2332.392, "p95_ms": 2332.392}, "triton_wait": {"avg_ms": 88.698, "count": 14, "max_ms": 105.194, "p95_ms": 105.194}, "visualize": {"avg_ms": 73.279, "count": 14, "max_ms": 87.198, "p95_ms": 87.198}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.614, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 213, "tracking_summary": 3080}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9473.523, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T06:27:25+09:00", "last_error_summary": "[h264 @ 0x564b95244000] error while decoding MB 5 37, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 255, "h264_decode_error_count": 439, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:27:25+09:00", "last_error_summary": "[h264 @ 0x564b95244000] error while decoding MB 5 37, bytestream -9", "last_exit_at": "2026-07-02T06:30:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 197.998, "last_started_at": "2026-07-02T06:27:29+09:00", "pid": 44401, "process": {"exists": false, "pid": 44401}, "restart_count": 285, "restart_count_per_hour": 27, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 286}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 2004.524, "sample_count": 181, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 369662, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:30:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T06:30:47+09:00", "last_frame_read_age_sec": 1.483, "last_frame_read_at": "2026-07-02T06:30:46+09:00", "mjpeg": {"age_sec": 1.471, "exists": true, "mtime": "2026-07-02T06:30:46.463597+09:00", "mtime_epoch": 1782941446.4635968, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 226148}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:30:47+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5534.594, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 49.1, "recorders": [{"age_sec": 49.1, "frame_count": 77, "is_recording": true, "is_stopping": false, "pid": 44508, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.234, "name": "ffmpeg", "pid": 44508, "rss_mb": 186.234, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 23, "returncode": null, "running": true, "started_at": "2026-07-02T06:29:58+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_062958_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [44508], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 18, "pushed_frames": 18, "read_frames": 17, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.604, "pushed": 0.604, "read": 0.57}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 142.292, "count": 17, "max_ms": 149.479, "p95_ms": 149.479}, "gstreamer_push": {"avg_ms": 0.853, "count": 18, "max_ms": 1.344, "p95_ms": 1.344}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.315, "count": 17, "max_ms": 67.916, "p95_ms": 67.916}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 84.879, "count": 17, "max_ms": 103.629, "p95_ms": 103.629}, "yolo11_640_fire_11class": {"avg_ms": 55.897, "count": 17, "max_ms": 70.483, "p95_ms": 70.483}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.886, "count": 17, "max_ms": 99.518, "p95_ms": 99.518}}, "truncated_count": 0}, "processing_total": {"avg_ms": 1747.939, "count": 18, "max_ms": 2124.327, "p95_ms": 2124.327}, "read_interval": {"avg_ms": 1773.153, "count": 17, "max_ms": 2133.901, "p95_ms": 2133.901}, "triton_wait": {"avg_ms": 84.892, "count": 17, "max_ms": 103.641, "p95_ms": 103.641}, "visualize": {"avg_ms": 23.396, "count": 17, "max_ms": 29.176, "p95_ms": 29.176}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.816, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 213, "tracking_summary": 3080}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9473.516, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 255, "h264_decode_error_count": 439, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:27:25+09:00", "last_error_summary": "[h264 @ 0x564b95244000] error while decoding MB 5 37, bytestream -9", "last_exit_at": "2026-07-02T06:30:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 198.0, "last_started_at": "2026-07-02T06:27:29+09:00", "pid": 44401, "process": {"exists": false, "pid": 44401}, "restart_count": 285, "restart_count_per_hour": 27, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 286}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 2004.524, "sample_count": 181, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 369662, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:30:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T06:30:47+09:00", "last_frame_read_age_sec": 1.485, "last_frame_read_at": "2026-07-02T06:30:46+09:00", "mjpeg": {"age_sec": 1.473, "exists": true, "mtime": "2026-07-02T06:30:46.463597+09:00", "mtime_epoch": 1782941446.4635968, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 226148}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:30:47+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5534.594, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 49.102, "recorders": [{"age_sec": 49.102, "frame_count": 77, "is_recording": true, "is_stopping": false, "pid": 44508, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.234, "name": "ffmpeg", "pid": 44508, "rss_mb": 186.234, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 23, "returncode": null, "running": true, "started_at": "2026-07-02T06:29:58+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_062958_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [44508], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 18, "pushed_frames": 18, "read_frames": 17, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.604, "pushed": 0.604, "read": 0.57}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 142.292, "count": 17, "max_ms": 149.479, "p95_ms": 149.479}, "gstreamer_push": {"avg_ms": 0.853, "count": 18, "max_ms": 1.344, "p95_ms": 1.344}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.315, "count": 17, "max_ms": 67.916, "p95_ms": 67.916}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 84.879, "count": 17, "max_ms": 103.629, "p95_ms": 103.629}, "yolo11_640_fire_11class": {"avg_ms": 55.897, "count": 17, "max_ms": 70.483, "p95_ms": 70.483}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.886, "count": 17, "max_ms": 99.518, "p95_ms": 99.518}}, "truncated_count": 0}, "processing_total": {"avg_ms": 1747.939, "count": 18, "max_ms": 2124.327, "p95_ms": 2124.327}, "read_interval": {"avg_ms": 1773.153, "count": 17, "max_ms": 2133.901, "p95_ms": 2133.901}, "triton_wait": {"avg_ms": 84.892, "count": 17, "max_ms": 103.641, "p95_ms": 103.641}, "visualize": {"avg_ms": 23.396, "count": 17, "max_ms": 29.176, "p95_ms": 29.176}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.818, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 213, "tracking_summary": 3080}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9576.664, "limit_mb": 8796093022207.996}, "detail": {"pid": 44568, "restart_count": 286, "start_count": 287}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 255, "h264_decode_error_count": 439, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:27:25+09:00", "last_error_summary": "[h264 @ 0x564b95244000] error while decoding MB 5 37, bytestream -9", "last_exit_at": "2026-07-02T06:30:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T06:30:50+09:00", "pid": 44568, "process": {"exists": true, "fd_count": 5, "hwm_mb": 133.047, "name": "ffmpeg", "pid": 44568, "rss_mb": 133.047, "state": "S (sleeping)", "threads": 106}, "restart_count": 286, "restart_count_per_hour": 28, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 287}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 2004.524, "sample_count": 178, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 369662, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:30:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.924, "last_frame_pushed_at": "2026-07-02T06:30:47+09:00", "last_frame_read_age_sec": 6.404, "last_frame_read_at": "2026-07-02T06:30:46+09:00", "mjpeg": {"age_sec": 4.904, "exists": true, "mtime": "2026-07-02T06:30:47.951611+09:00", "mtime_epoch": 1782941447.9516106, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 226248}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:30:52+09:00", "process": {"exists": true, "fd_count": 123, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5534.594, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 54.022, "recorders": [{"age_sec": 54.022, "frame_count": 77, "is_recording": true, "is_stopping": false, "pid": 44508, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.234, "name": "ffmpeg", "pid": 44508, "rss_mb": 186.234, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 23, "returncode": null, "running": true, "started_at": "2026-07-02T06:29:58+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_062958_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [44508], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 15, "pushed_frames": 15, "read_frames": 14, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.505, "pushed": 0.505, "read": 0.472}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 142.409, "count": 14, "max_ms": 149.479, "p95_ms": 149.479}, "gstreamer_push": {"avg_ms": 0.86, "count": 15, "max_ms": 1.344, "p95_ms": 1.344}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.6, "count": 14, "max_ms": 67.916, "p95_ms": 67.916}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 85.827, "count": 14, "max_ms": 103.629, "p95_ms": 103.629}, "yolo11_640_fire_11class": {"avg_ms": 56.672, "count": 14, "max_ms": 70.483, "p95_ms": 70.483}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.139, "count": 14, "max_ms": 99.518, "p95_ms": 99.518}}, "truncated_count": 0}, "processing_total": {"avg_ms": 1737.419, "count": 15, "max_ms": 2124.327, "p95_ms": 2124.327}, "read_interval": {"avg_ms": 1765.331, "count": 14, "max_ms": 2133.901, "p95_ms": 2133.901}, "triton_wait": {"avg_ms": 85.84, "count": 14, "max_ms": 103.641, "p95_ms": 103.641}, "visualize": {"avg_ms": 23.424, "count": 14, "max_ms": 29.176, "p95_ms": 29.176}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.675, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 213, "tracking_summary": 3083}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9324.281, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55e33d9e3a00] top block unavailable for requested intra mode -1 [h264 @ 0x55e33d9e3a00] error while decoding MB 74 0, bytestream 1235", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 256, "h264_decode_error_count": 440, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:32:52+09:00", "last_error_summary": "[h264 @ 0x55e33d9e3a00] top block unavailable for requested intra mode -1 [h264 @ 0x55e33d9e3a00] error while decoding MB 74 0, bytestream 1235", "last_exit_at": "2026-07-02T06:30:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.247, "last_started_at": "2026-07-02T06:30:50+09:00", "pid": 44568, "process": {"exists": false, "pid": 44568}, "restart_count": 286, "restart_count_per_hour": 26, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 287}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1796.631, "sample_count": 446, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 369997, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:32:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-02T06:32:52+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-02T06:32:52+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-02T06:32:52.056862+09:00", "mtime_epoch": 1782941572.0568619, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206008}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:32:52+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5534.594, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 85, "pushed_frames": 85, "read_frames": 85, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.862, "pushed": 2.862, "read": 2.862}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.196, "count": 85, "max_ms": 32.861, "p95_ms": 30.633}, "gstreamer_push": {"avg_ms": 0.878, "count": 85, "max_ms": 1.227, "p95_ms": 0.932}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.036, "count": 85, "max_ms": 74.489, "p95_ms": 71.853}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.61, "count": 85, "max_ms": 101.211, "p95_ms": 99.601}, "yolo11_640_fire_11class": {"avg_ms": 55.671, "count": 85, "max_ms": 72.133, "p95_ms": 70.693}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.348, "count": 85, "max_ms": 102.121, "p95_ms": 97.302}}, "truncated_count": 0}, "processing_total": {"avg_ms": 343.842, "count": 85, "max_ms": 411.758, "p95_ms": 394.684}, "read_interval": {"avg_ms": 353.163, "count": 85, "max_ms": 422.659, "p95_ms": 403.4}, "triton_wait": {"avg_ms": 77.875, "count": 85, "max_ms": 102.133, "p95_ms": 99.614}, "visualize": {"avg_ms": 12.089, "count": 85, "max_ms": 15.153, "p95_ms": 14.454}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.702, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 213, "tracking_summary": 3083}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9323.324, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T06:32:52+09:00", "last_error_summary": "[h264 @ 0x55e33d9e3a00] top block unavailable for requested intra mode -1 [h264 @ 0x55e33d9e3a00] error while decoding MB 74 0, bytestream 1235", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 256, "h264_decode_error_count": 440, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:32:52+09:00", "last_error_summary": "[h264 @ 0x55e33d9e3a00] top block unavailable for requested intra mode -1 [h264 @ 0x55e33d9e3a00] error while decoding MB 74 0, bytestream 1235", "last_exit_at": "2026-07-02T06:32:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.539, "last_started_at": "2026-07-02T06:30:50+09:00", "pid": 44568, "process": {"exists": false, "pid": 44568}, "restart_count": 286, "restart_count_per_hour": 26, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 287}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1796.631, "sample_count": 447, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 369998, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:32:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T06:32:52+09:00", "last_frame_read_age_sec": 0.344, "last_frame_read_at": "2026-07-02T06:32:52+09:00", "mjpeg": {"age_sec": 0.337, "exists": true, "mtime": "2026-07-02T06:32:52.056862+09:00", "mtime_epoch": 1782941572.0568619, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206008}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:32:52+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5534.594, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 86, "pushed_frames": 86, "read_frames": 85, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.867, "pushed": 2.867, "read": 2.834}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.2, "count": 85, "max_ms": 32.861, "p95_ms": 30.633}, "gstreamer_push": {"avg_ms": 0.878, "count": 86, "max_ms": 1.227, "p95_ms": 0.932}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.929, "count": 85, "max_ms": 74.489, "p95_ms": 71.853}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.403, "count": 85, "max_ms": 101.211, "p95_ms": 97.706}, "yolo11_640_fire_11class": {"avg_ms": 55.585, "count": 85, "max_ms": 72.133, "p95_ms": 70.693}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.961, "count": 85, "max_ms": 102.121, "p95_ms": 97.147}}, "truncated_count": 0}, "processing_total": {"avg_ms": 343.784, "count": 86, "max_ms": 411.758, "p95_ms": 394.684}, "read_interval": {"avg_ms": 353.163, "count": 85, "max_ms": 422.659, "p95_ms": 403.4}, "triton_wait": {"avg_ms": 77.655, "count": 85, "max_ms": 102.133, "p95_ms": 97.718}, "visualize": {"avg_ms": 12.065, "count": 85, "max_ms": 15.153, "p95_ms": 14.454}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 213, "tracking_summary": 3083}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9323.562, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 256, "h264_decode_error_count": 440, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:32:52+09:00", "last_error_summary": "[h264 @ 0x55e33d9e3a00] top block unavailable for requested intra mode -1 [h264 @ 0x55e33d9e3a00] error while decoding MB 74 0, bytestream 1235", "last_exit_at": "2026-07-02T06:32:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.542, "last_started_at": "2026-07-02T06:30:50+09:00", "pid": 44568, "process": {"exists": false, "pid": 44568}, "restart_count": 286, "restart_count_per_hour": 26, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 287}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1796.631, "sample_count": 447, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 369998, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:32:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T06:32:52+09:00", "last_frame_read_age_sec": 0.347, "last_frame_read_at": "2026-07-02T06:32:52+09:00", "mjpeg": {"age_sec": 0.34, "exists": true, "mtime": "2026-07-02T06:32:52.056862+09:00", "mtime_epoch": 1782941572.0568619, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206008}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:32:52+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5534.594, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 86, "pushed_frames": 86, "read_frames": 85, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.867, "pushed": 2.867, "read": 2.834}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.2, "count": 85, "max_ms": 32.861, "p95_ms": 30.633}, "gstreamer_push": {"avg_ms": 0.878, "count": 86, "max_ms": 1.227, "p95_ms": 0.932}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.929, "count": 85, "max_ms": 74.489, "p95_ms": 71.853}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.403, "count": 85, "max_ms": 101.211, "p95_ms": 97.706}, "yolo11_640_fire_11class": {"avg_ms": 55.585, "count": 85, "max_ms": 72.133, "p95_ms": 70.693}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.961, "count": 85, "max_ms": 102.121, "p95_ms": 97.147}}, "truncated_count": 0}, "processing_total": {"avg_ms": 343.784, "count": 86, "max_ms": 411.758, "p95_ms": 394.684}, "read_interval": {"avg_ms": 353.163, "count": 85, "max_ms": 422.659, "p95_ms": 403.4}, "triton_wait": {"avg_ms": 77.655, "count": 85, "max_ms": 102.133, "p95_ms": 97.718}, "visualize": {"avg_ms": 12.065, "count": 85, "max_ms": 15.153, "p95_ms": 14.454}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 213, "tracking_summary": 3083}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9426.0, "limit_mb": 8796093022207.996}, "detail": {"pid": 44675, "restart_count": 287, "start_count": 288}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 256, "h264_decode_error_count": 440, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:32:52+09:00", "last_error_summary": "[h264 @ 0x55e33d9e3a00] top block unavailable for requested intra mode -1 [h264 @ 0x55e33d9e3a00] error while decoding MB 74 0, bytestream 1235", "last_exit_at": "2026-07-02T06:32:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T06:32:55+09:00", "pid": 44675, "process": {"exists": true, "fd_count": 5, "hwm_mb": 133.875, "name": "ffmpeg", "pid": 44675, "rss_mb": 133.875, "state": "S (sleeping)", "threads": 106}, "restart_count": 287, "restart_count_per_hour": 27, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 288}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1796.631, "sample_count": 444, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 369998, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:32:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.971, "last_frame_pushed_at": "2026-07-02T06:32:52+09:00", "last_frame_read_age_sec": 5.311, "last_frame_read_at": "2026-07-02T06:32:52+09:00", "mjpeg": {"age_sec": 4.956, "exists": true, "mtime": "2026-07-02T06:32:52.404866+09:00", "mtime_epoch": 1782941572.4048657, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206344}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:32:57+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5534.594, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 72, "pushed_frames": 72, "read_frames": 71, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.422, "pushed": 2.422, "read": 2.388}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.199, "count": 72, "max_ms": 32.861, "p95_ms": 30.633}, "gstreamer_push": {"avg_ms": 0.879, "count": 72, "max_ms": 1.227, "p95_ms": 0.932}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.132, "count": 72, "max_ms": 74.489, "p95_ms": 72.018}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.786, "count": 72, "max_ms": 100.958, "p95_ms": 95.344}, "yolo11_640_fire_11class": {"avg_ms": 55.414, "count": 72, "max_ms": 72.133, "p95_ms": 70.536}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.579, "count": 72, "max_ms": 102.121, "p95_ms": 88.665}}, "truncated_count": 0}, "processing_total": {"avg_ms": 340.328, "count": 72, "max_ms": 411.758, "p95_ms": 395.594}, "read_interval": {"avg_ms": 349.684, "count": 71, "max_ms": 422.659, "p95_ms": 404.246}, "triton_wait": {"avg_ms": 77.003, "count": 72, "max_ms": 102.133, "p95_ms": 95.359}, "visualize": {"avg_ms": 11.844, "count": 72, "max_ms": 14.929, "p95_ms": 14.426}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.727, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 213, "tracking_summary": 3086}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9325.762, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5623315aca40] error while decoding MB 61 8, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 257, "h264_decode_error_count": 441, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:35:05+09:00", "last_error_summary": "[h264 @ 0x5623315aca40] error while decoding MB 61 8, bytestream -5", "last_exit_at": "2026-07-02T06:32:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 130.126, "last_started_at": "2026-07-02T06:32:55+09:00", "pid": 44675, "process": {"exists": false, "pid": 44675}, "restart_count": 287, "restart_count_per_hour": 26, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 288}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 397.324, "sample_count": 753, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 370391, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:35:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-02T06:35:05+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-02T06:35:05+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-02T06:35:05.446410+09:00", "mtime_epoch": 1782941705.44641, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205560}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:35:05+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5534.594, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 94, "pushed_frames": 94, "read_frames": 94, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.149, "pushed": 3.149, "read": 3.149}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 21.135, "count": 93, "max_ms": 29.118, "p95_ms": 24.472}, "gstreamer_push": {"avg_ms": 0.867, "count": 94, "max_ms": 0.915, "p95_ms": 0.903}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.002, "count": 93, "max_ms": 66.306, "p95_ms": 60.665}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.36, "count": 93, "max_ms": 102.73, "p95_ms": 98.886}, "yolo11_640_fire_11class": {"avg_ms": 53.606, "count": 93, "max_ms": 72.489, "p95_ms": 69.507}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.688, "count": 93, "max_ms": 99.939, "p95_ms": 97.491}}, "truncated_count": 0}, "processing_total": {"avg_ms": 310.621, "count": 94, "max_ms": 408.014, "p95_ms": 380.817}, "read_interval": {"avg_ms": 320.009, "count": 94, "max_ms": 418.527, "p95_ms": 389.693}, "triton_wait": {"avg_ms": 79.906, "count": 93, "max_ms": 102.743, "p95_ms": 98.895}, "visualize": {"avg_ms": 11.836, "count": 93, "max_ms": 15.537, "p95_ms": 14.752}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.846, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 213, "tracking_summary": 3086}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9324.223, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T06:35:05+09:00", "last_error_summary": "[h264 @ 0x5623315aca40] error while decoding MB 61 8, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 257, "h264_decode_error_count": 441, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:35:05+09:00", "last_error_summary": "[h264 @ 0x5623315aca40] error while decoding MB 61 8, bytestream -5", "last_exit_at": "2026-07-02T06:35:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 130.367, "last_started_at": "2026-07-02T06:32:55+09:00", "pid": 44675, "process": {"exists": false, "pid": 44675}, "restart_count": 287, "restart_count_per_hour": 26, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 288}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 397.324, "sample_count": 754, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 370392, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:35:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T06:35:05+09:00", "last_frame_read_age_sec": 0.292, "last_frame_read_at": "2026-07-02T06:35:05+09:00", "mjpeg": {"age_sec": 0.28, "exists": true, "mtime": "2026-07-02T06:35:05.446410+09:00", "mtime_epoch": 1782941705.44641, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205560}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:35:05+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5534.594, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 94, "pushed_frames": 94, "read_frames": 93, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.156, "pushed": 3.156, "read": 3.122}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 21.116, "count": 94, "max_ms": 29.118, "p95_ms": 24.472}, "gstreamer_push": {"avg_ms": 0.867, "count": 94, "max_ms": 0.915, "p95_ms": 0.903}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.002, "count": 94, "max_ms": 66.306, "p95_ms": 60.665}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.548, "count": 94, "max_ms": 102.73, "p95_ms": 98.886}, "yolo11_640_fire_11class": {"avg_ms": 53.745, "count": 94, "max_ms": 72.489, "p95_ms": 69.507}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.855, "count": 94, "max_ms": 99.939, "p95_ms": 97.491}}, "truncated_count": 0}, "processing_total": {"avg_ms": 310.642, "count": 94, "max_ms": 408.014, "p95_ms": 380.817}, "read_interval": {"avg_ms": 320.285, "count": 93, "max_ms": 418.527, "p95_ms": 389.693}, "triton_wait": {"avg_ms": 80.078, "count": 94, "max_ms": 102.743, "p95_ms": 98.895}, "visualize": {"avg_ms": 11.817, "count": 94, "max_ms": 15.537, "p95_ms": 14.752}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.789, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 213, "tracking_summary": 3086}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9324.215, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 257, "h264_decode_error_count": 441, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:35:05+09:00", "last_error_summary": "[h264 @ 0x5623315aca40] error while decoding MB 61 8, bytestream -5", "last_exit_at": "2026-07-02T06:35:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 130.37, "last_started_at": "2026-07-02T06:32:55+09:00", "pid": 44675, "process": {"exists": false, "pid": 44675}, "restart_count": 287, "restart_count_per_hour": 26, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 288}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 397.324, "sample_count": 754, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 370392, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:35:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T06:35:05+09:00", "last_frame_read_age_sec": 0.295, "last_frame_read_at": "2026-07-02T06:35:05+09:00", "mjpeg": {"age_sec": 0.283, "exists": true, "mtime": "2026-07-02T06:35:05.446410+09:00", "mtime_epoch": 1782941705.44641, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205560}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:35:05+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5534.594, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 94, "pushed_frames": 94, "read_frames": 93, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.155, "pushed": 3.155, "read": 3.122}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 21.116, "count": 94, "max_ms": 29.118, "p95_ms": 24.472}, "gstreamer_push": {"avg_ms": 0.867, "count": 94, "max_ms": 0.915, "p95_ms": 0.903}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.002, "count": 94, "max_ms": 66.306, "p95_ms": 60.665}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.548, "count": 94, "max_ms": 102.73, "p95_ms": 98.886}, "yolo11_640_fire_11class": {"avg_ms": 53.745, "count": 94, "max_ms": 72.489, "p95_ms": 69.507}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.855, "count": 94, "max_ms": 99.939, "p95_ms": 97.491}}, "truncated_count": 0}, "processing_total": {"avg_ms": 310.642, "count": 94, "max_ms": 408.014, "p95_ms": 380.817}, "read_interval": {"avg_ms": 320.285, "count": 93, "max_ms": 418.527, "p95_ms": 389.693}, "triton_wait": {"avg_ms": 80.078, "count": 94, "max_ms": 102.743, "p95_ms": 98.895}, "visualize": {"avg_ms": 11.817, "count": 94, "max_ms": 15.537, "p95_ms": 14.752}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.792, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 213, "tracking_summary": 3086}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9438.176, "limit_mb": 8796093022207.996}, "detail": {"pid": 44782, "restart_count": 288, "start_count": 289}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 257, "h264_decode_error_count": 441, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:35:05+09:00", "last_error_summary": "[h264 @ 0x5623315aca40] error while decoding MB 61 8, bytestream -5", "last_exit_at": "2026-07-02T06:35:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T06:35:08+09:00", "pid": 44782, "process": {"exists": true, "fd_count": 5, "hwm_mb": 144.645, "name": "ffmpeg", "pid": 44782, "rss_mb": 144.645, "state": "S (sleeping)", "threads": 106}, "restart_count": 288, "restart_count_per_hour": 27, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 289}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 396.821, "sample_count": 752, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 370392, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:35:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.466, "last_frame_pushed_at": "2026-07-02T06:35:05+09:00", "last_frame_read_age_sec": 4.754, "last_frame_read_at": "2026-07-02T06:35:05+09:00", "mjpeg": {"age_sec": 4.443, "exists": true, "mtime": "2026-07-02T06:35:05.746413+09:00", "mtime_epoch": 1782941705.7464135, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205874}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:35:10+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5534.594, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 79, "pushed_frames": 79, "read_frames": 78, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.637, "pushed": 2.637, "read": 2.604}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 21.396, "count": 78, "max_ms": 29.118, "p95_ms": 24.969}, "gstreamer_push": {"avg_ms": 0.866, "count": 79, "max_ms": 0.915, "p95_ms": 0.903}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.62, "count": 78, "max_ms": 62.232, "p95_ms": 59.556}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.38, "count": 78, "max_ms": 102.73, "p95_ms": 98.886}, "yolo11_640_fire_11class": {"avg_ms": 53.247, "count": 78, "max_ms": 72.489, "p95_ms": 68.903}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.139, "count": 78, "max_ms": 99.939, "p95_ms": 96.928}}, "truncated_count": 0}, "processing_total": {"avg_ms": 317.068, "count": 79, "max_ms": 408.014, "p95_ms": 383.987}, "read_interval": {"avg_ms": 326.794, "count": 78, "max_ms": 418.527, "p95_ms": 394.701}, "triton_wait": {"avg_ms": 79.154, "count": 78, "max_ms": 102.743, "p95_ms": 98.895}, "visualize": {"avg_ms": 12.205, "count": 78, "max_ms": 15.537, "p95_ms": 14.787}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.953, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 215, "tracking_summary": 3087}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9492.914, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55bbc3fb0540] error while decoding MB 47 37, bytestream -13", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 258, "h264_decode_error_count": 442, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:38:09+09:00", "last_error_summary": "[h264 @ 0x55bbc3fb0540] error while decoding MB 47 37, bytestream -13", "last_exit_at": "2026-07-02T06:35:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 181.734, "last_started_at": "2026-07-02T06:35:08+09:00", "pid": 44782, "process": {"exists": false, "pid": 44782}, "restart_count": 288, "restart_count_per_hour": 26, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 289}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1479.297, "sample_count": 485, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 370520, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:38:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-02T06:38:09+09:00", "last_frame_read_age_sec": 0.056, "last_frame_read_at": "2026-07-02T06:38:09+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-02T06:38:09.880810+09:00", "mtime_epoch": 1782941889.8808103, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 216659}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:38:09+09:00", "process": {"exists": true, "fd_count": 123, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5534.852, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 1.912, "recorders": [{"age_sec": 1.912, "frame_count": 51, "is_recording": true, "is_stopping": false, "pid": 44949, "process": {"exists": true, "fd_count": 5, "hwm_mb": 187.922, "name": "ffmpeg", "pid": 44949, "rss_mb": 187.922, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 49, "returncode": null, "running": true, "started_at": "2026-07-02T06:38:08+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_063807_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [44949], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 19, "pushed_frames": 19, "read_frames": 19, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.643, "pushed": 0.643, "read": 0.643}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 149.314, "count": 18, "max_ms": 165.945, "p95_ms": 165.945}, "gstreamer_push": {"avg_ms": 0.731, "count": 19, "max_ms": 0.882, "p95_ms": 0.882}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.052, "count": 18, "max_ms": 67.034, "p95_ms": 67.034}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.829, "count": 18, "max_ms": 103.918, "p95_ms": 103.918}, "yolo11_640_fire_11class": {"avg_ms": 52.145, "count": 18, "max_ms": 68.702, "p95_ms": 68.702}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.481, "count": 18, "max_ms": 118.975, "p95_ms": 118.975}}, "truncated_count": 0}, "processing_total": {"avg_ms": 1617.615, "count": 19, "max_ms": 2100.231, "p95_ms": 2100.231}, "read_interval": {"avg_ms": 1626.521, "count": 19, "max_ms": 2108.465, "p95_ms": 2108.465}, "triton_wait": {"avg_ms": 83.571, "count": 18, "max_ms": 119.017, "p95_ms": 119.017}, "visualize": {"avg_ms": 24.037, "count": 18, "max_ms": 28.157, "p95_ms": 28.157}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.527, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 215, "tracking_summary": 3087}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9491.309, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T06:38:09+09:00", "last_error_summary": "[h264 @ 0x55bbc3fb0540] error while decoding MB 47 37, bytestream -13", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 258, "h264_decode_error_count": 442, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:38:09+09:00", "last_error_summary": "[h264 @ 0x55bbc3fb0540] error while decoding MB 47 37, bytestream -13", "last_exit_at": "2026-07-02T06:38:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 183.136, "last_started_at": "2026-07-02T06:35:08+09:00", "pid": 44782, "process": {"exists": false, "pid": 44782}, "restart_count": 288, "restart_count_per_hour": 26, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 289}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1479.297, "sample_count": 481, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 370521, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:38:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T06:38:11+09:00", "last_frame_read_age_sec": 1.458, "last_frame_read_at": "2026-07-02T06:38:09+09:00", "mjpeg": {"age_sec": 1.444, "exists": true, "mtime": "2026-07-02T06:38:09.880810+09:00", "mtime_epoch": 1782941889.8808103, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 216659}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:38:11+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5534.852, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 3.313, "recorders": [{"age_sec": 3.313, "frame_count": 52, "is_recording": true, "is_stopping": false, "pid": 44949, "process": {"exists": true, "fd_count": 5, "hwm_mb": 187.922, "name": "ffmpeg", "pid": 44949, "rss_mb": 187.922, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 48, "returncode": null, "running": true, "started_at": "2026-07-02T06:38:08+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_063807_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [44949], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 19, "pushed_frames": 19, "read_frames": 18, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.644, "pushed": 0.644, "read": 0.61}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 149.302, "count": 18, "max_ms": 165.945, "p95_ms": 165.945}, "gstreamer_push": {"avg_ms": 0.733, "count": 19, "max_ms": 0.882, "p95_ms": 0.882}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.643, "count": 18, "max_ms": 67.034, "p95_ms": 67.034}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.192, "count": 18, "max_ms": 103.918, "p95_ms": 103.918}, "yolo11_640_fire_11class": {"avg_ms": 52.342, "count": 18, "max_ms": 68.702, "p95_ms": 68.702}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.968, "count": 18, "max_ms": 118.975, "p95_ms": 118.975}}, "truncated_count": 0}, "processing_total": {"avg_ms": 1618.751, "count": 19, "max_ms": 2100.231, "p95_ms": 2100.231}, "read_interval": {"avg_ms": 1636.851, "count": 18, "max_ms": 2108.465, "p95_ms": 2108.465}, "triton_wait": {"avg_ms": 83.935, "count": 18, "max_ms": 119.017, "p95_ms": 119.017}, "visualize": {"avg_ms": 24.051, "count": 18, "max_ms": 28.157, "p95_ms": 28.157}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.507, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 215, "tracking_summary": 3087}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9491.301, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 258, "h264_decode_error_count": 442, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:38:09+09:00", "last_error_summary": "[h264 @ 0x55bbc3fb0540] error while decoding MB 47 37, bytestream -13", "last_exit_at": "2026-07-02T06:38:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 183.139, "last_started_at": "2026-07-02T06:35:08+09:00", "pid": 44782, "process": {"exists": false, "pid": 44782}, "restart_count": 288, "restart_count_per_hour": 26, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 289}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1479.297, "sample_count": 481, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 370521, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:38:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T06:38:11+09:00", "last_frame_read_age_sec": 1.46, "last_frame_read_at": "2026-07-02T06:38:09+09:00", "mjpeg": {"age_sec": 1.447, "exists": true, "mtime": "2026-07-02T06:38:09.880810+09:00", "mtime_epoch": 1782941889.8808103, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 216659}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:38:11+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5534.852, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 3.315, "recorders": [{"age_sec": 3.315, "frame_count": 52, "is_recording": true, "is_stopping": false, "pid": 44949, "process": {"exists": true, "fd_count": 5, "hwm_mb": 187.922, "name": "ffmpeg", "pid": 44949, "rss_mb": 187.922, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 48, "returncode": null, "running": true, "started_at": "2026-07-02T06:38:08+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_063807_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [44949], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 19, "pushed_frames": 19, "read_frames": 18, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.644, "pushed": 0.644, "read": 0.61}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 149.302, "count": 18, "max_ms": 165.945, "p95_ms": 165.945}, "gstreamer_push": {"avg_ms": 0.733, "count": 19, "max_ms": 0.882, "p95_ms": 0.882}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.643, "count": 18, "max_ms": 67.034, "p95_ms": 67.034}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.192, "count": 18, "max_ms": 103.918, "p95_ms": 103.918}, "yolo11_640_fire_11class": {"avg_ms": 52.342, "count": 18, "max_ms": 68.702, "p95_ms": 68.702}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.968, "count": 18, "max_ms": 118.975, "p95_ms": 118.975}}, "truncated_count": 0}, "processing_total": {"avg_ms": 1618.751, "count": 19, "max_ms": 2100.231, "p95_ms": 2100.231}, "read_interval": {"avg_ms": 1636.851, "count": 18, "max_ms": 2108.465, "p95_ms": 2108.465}, "triton_wait": {"avg_ms": 83.935, "count": 18, "max_ms": 119.017, "p95_ms": 119.017}, "visualize": {"avg_ms": 24.051, "count": 18, "max_ms": 28.157, "p95_ms": 28.157}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.509, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 215, "tracking_summary": 3087}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9591.641, "limit_mb": 8796093022207.996}, "detail": {"pid": 45009, "restart_count": 289, "start_count": 290}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 258, "h264_decode_error_count": 442, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:38:09+09:00", "last_error_summary": "[h264 @ 0x55bbc3fb0540] error while decoding MB 47 37, bytestream -13", "last_exit_at": "2026-07-02T06:38:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T06:38:13+09:00", "pid": 45009, "process": {"exists": true, "fd_count": 5, "hwm_mb": 130.707, "name": "ffmpeg", "pid": 45009, "rss_mb": 130.707, "state": "S (sleeping)", "threads": 106}, "restart_count": 289, "restart_count_per_hour": 27, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 290}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1480.038, "sample_count": 465, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 370521, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:38:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.54, "last_frame_pushed_at": "2026-07-02T06:38:11+09:00", "last_frame_read_age_sec": 5.995, "last_frame_read_at": "2026-07-02T06:38:09+09:00", "mjpeg": {"age_sec": 4.522, "exists": true, "mtime": "2026-07-02T06:38:11.340830+09:00", "mtime_epoch": 1782941891.34083, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 215945}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:38:15+09:00", "process": {"exists": true, "fd_count": 123, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5534.852, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 7.851, "recorders": [{"age_sec": 7.851, "frame_count": 52, "is_recording": true, "is_stopping": false, "pid": 44949, "process": {"exists": true, "fd_count": 5, "hwm_mb": 187.922, "name": "ffmpeg", "pid": 44949, "rss_mb": 187.922, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 48, "returncode": null, "running": true, "started_at": "2026-07-02T06:38:08+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_063807_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [44949], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 16, "pushed_frames": 16, "read_frames": 15, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.554, "pushed": 0.554, "read": 0.519}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 151.182, "count": 15, "max_ms": 165.945, "p95_ms": 165.945}, "gstreamer_push": {"avg_ms": 0.722, "count": 16, "max_ms": 0.882, "p95_ms": 0.882}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.145, "count": 15, "max_ms": 65.945, "p95_ms": 65.945}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.544, "count": 15, "max_ms": 103.918, "p95_ms": 103.918}, "yolo11_640_fire_11class": {"avg_ms": 51.088, "count": 15, "max_ms": 67.927, "p95_ms": 67.927}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.337, "count": 15, "max_ms": 106.609, "p95_ms": 106.609}}, "truncated_count": 0}, "processing_total": {"avg_ms": 1625.266, "count": 16, "max_ms": 2100.231, "p95_ms": 2100.231}, "read_interval": {"avg_ms": 1645.5, "count": 15, "max_ms": 2108.465, "p95_ms": 2108.465}, "triton_wait": {"avg_ms": 81.414, "count": 15, "max_ms": 106.621, "p95_ms": 106.621}, "visualize": {"avg_ms": 23.735, "count": 15, "max_ms": 28.157, "p95_ms": 28.157}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 28.904, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 222, "tracking_summary": 3088}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9513.738, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5642d84b2280] error while decoding MB 79 28, bytestream -6", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 259, "h264_decode_error_count": 443, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:40:32+09:00", "last_error_summary": "[h264 @ 0x5642d84b2280] error while decoding MB 79 28, bytestream -6", "last_exit_at": "2026-07-02T06:38:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 138.449, "last_started_at": "2026-07-02T06:38:13+09:00", "pid": 45009, "process": {"exists": false, "pid": 45009}, "restart_count": 289, "restart_count_per_hour": 25, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 290}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1608.831, "sample_count": 258, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 370665, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:40:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.056, "last_frame_pushed_at": "2026-07-02T06:40:32+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-02T06:40:32+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-02T06:40:32.270816+09:00", "mtime_epoch": 1782942032.2708163, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204071}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:40:32+09:00", "process": {"exists": true, "fd_count": 123, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5534.852, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 38.778, "recorders": [{"age_sec": 38.778, "frame_count": 89, "is_recording": true, "is_stopping": false, "pid": 45116, "process": {"exists": true, "fd_count": 5, "hwm_mb": 187.137, "name": "ffmpeg", "pid": 45116, "rss_mb": 187.137, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 11, "returncode": null, "running": true, "started_at": "2026-07-02T06:39:53+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_063953_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [45116], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 30, "pushed_frames": 30, "read_frames": 30, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.025, "pushed": 1.025, "read": 1.025}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 74.889, "count": 29, "max_ms": 83.021, "p95_ms": 82.0}, "gstreamer_push": {"avg_ms": 0.895, "count": 30, "max_ms": 1.003, "p95_ms": 0.964}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.505, "count": 29, "max_ms": 61.008, "p95_ms": 60.349}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 87.302, "count": 29, "max_ms": 106.492, "p95_ms": 105.464}, "yolo11_640_fire_11class": {"avg_ms": 52.524, "count": 29, "max_ms": 70.043, "p95_ms": 69.762}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.254, "count": 29, "max_ms": 100.137, "p95_ms": 97.918}}, "truncated_count": 0}, "processing_total": {"avg_ms": 995.264, "count": 30, "max_ms": 1198.998, "p95_ms": 1080.033}, "read_interval": {"avg_ms": 1004.984, "count": 30, "max_ms": 1208.485, "p95_ms": 1089.966}, "triton_wait": {"avg_ms": 87.316, "count": 29, "max_ms": 106.506, "p95_ms": 105.477}, "visualize": {"avg_ms": 17.155, "count": 29, "max_ms": 21.409, "p95_ms": 20.537}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.277, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 222, "tracking_summary": 3088}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9509.336, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T06:40:32+09:00", "last_error_summary": "[h264 @ 0x5642d84b2280] error while decoding MB 79 28, bytestream -6", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 259, "h264_decode_error_count": 443, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:40:32+09:00", "last_error_summary": "[h264 @ 0x5642d84b2280] error while decoding MB 79 28, bytestream -6", "last_exit_at": "2026-07-02T06:40:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 139.374, "last_started_at": "2026-07-02T06:38:13+09:00", "pid": 45009, "process": {"exists": false, "pid": 45009}, "restart_count": 289, "restart_count_per_hour": 25, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 290}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1608.831, "sample_count": 258, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 370666, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:40:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T06:40:33+09:00", "last_frame_read_age_sec": 0.974, "last_frame_read_at": "2026-07-02T06:40:32+09:00", "mjpeg": {"age_sec": 0.965, "exists": true, "mtime": "2026-07-02T06:40:32.270816+09:00", "mtime_epoch": 1782942032.2708163, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204071}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:40:33+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5534.852, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 39.702, "recorders": [{"age_sec": 39.702, "frame_count": 90, "is_recording": true, "is_stopping": false, "pid": 45116, "process": {"exists": true, "fd_count": 5, "hwm_mb": 187.137, "name": "ffmpeg", "pid": 45116, "rss_mb": 187.137, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 10, "returncode": null, "running": true, "started_at": "2026-07-02T06:39:53+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_063953_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [45116], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 30, "pushed_frames": 30, "read_frames": 29, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.025, "pushed": 1.025, "read": 0.991}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 74.957, "count": 29, "max_ms": 83.021, "p95_ms": 82.0}, "gstreamer_push": {"avg_ms": 0.893, "count": 30, "max_ms": 1.003, "p95_ms": 0.964}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.19, "count": 29, "max_ms": 61.008, "p95_ms": 60.349}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 87.112, "count": 29, "max_ms": 106.492, "p95_ms": 105.464}, "yolo11_640_fire_11class": {"avg_ms": 52.227, "count": 29, "max_ms": 70.043, "p95_ms": 69.762}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.986, "count": 29, "max_ms": 100.137, "p95_ms": 97.918}}, "truncated_count": 0}, "processing_total": {"avg_ms": 996.843, "count": 30, "max_ms": 1198.998, "p95_ms": 1080.033}, "read_interval": {"avg_ms": 1007.566, "count": 29, "max_ms": 1208.485, "p95_ms": 1089.966}, "triton_wait": {"avg_ms": 87.126, "count": 29, "max_ms": 106.506, "p95_ms": 105.477}, "visualize": {"avg_ms": 17.095, "count": 29, "max_ms": 21.409, "p95_ms": 20.537}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.274, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 222, "tracking_summary": 3088}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9509.273, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 259, "h264_decode_error_count": 443, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:40:32+09:00", "last_error_summary": "[h264 @ 0x5642d84b2280] error while decoding MB 79 28, bytestream -6", "last_exit_at": "2026-07-02T06:40:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 139.377, "last_started_at": "2026-07-02T06:38:13+09:00", "pid": 45009, "process": {"exists": false, "pid": 45009}, "restart_count": 289, "restart_count_per_hour": 25, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 290}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1608.831, "sample_count": 258, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 370666, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:40:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T06:40:33+09:00", "last_frame_read_age_sec": 0.976, "last_frame_read_at": "2026-07-02T06:40:32+09:00", "mjpeg": {"age_sec": 0.968, "exists": true, "mtime": "2026-07-02T06:40:32.270816+09:00", "mtime_epoch": 1782942032.2708163, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204071}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:40:33+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5534.852, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 39.704, "recorders": [{"age_sec": 39.704, "frame_count": 90, "is_recording": true, "is_stopping": false, "pid": 45116, "process": {"exists": true, "fd_count": 5, "hwm_mb": 187.137, "name": "ffmpeg", "pid": 45116, "rss_mb": 187.137, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 10, "returncode": null, "running": true, "started_at": "2026-07-02T06:39:53+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_063953_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [45116], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 30, "pushed_frames": 30, "read_frames": 29, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.025, "pushed": 1.025, "read": 0.991}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 74.957, "count": 29, "max_ms": 83.021, "p95_ms": 82.0}, "gstreamer_push": {"avg_ms": 0.893, "count": 30, "max_ms": 1.003, "p95_ms": 0.964}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.19, "count": 29, "max_ms": 61.008, "p95_ms": 60.349}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 87.112, "count": 29, "max_ms": 106.492, "p95_ms": 105.464}, "yolo11_640_fire_11class": {"avg_ms": 52.227, "count": 29, "max_ms": 70.043, "p95_ms": 69.762}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.986, "count": 29, "max_ms": 100.137, "p95_ms": 97.918}}, "truncated_count": 0}, "processing_total": {"avg_ms": 996.843, "count": 30, "max_ms": 1198.998, "p95_ms": 1080.033}, "read_interval": {"avg_ms": 1007.566, "count": 29, "max_ms": 1208.485, "p95_ms": 1089.966}, "triton_wait": {"avg_ms": 87.126, "count": 29, "max_ms": 106.506, "p95_ms": 105.477}, "visualize": {"avg_ms": 17.095, "count": 29, "max_ms": 21.409, "p95_ms": 20.537}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.277, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 222, "tracking_summary": 3088}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9618.297, "limit_mb": 8796093022207.996}, "detail": {"pid": 45176, "restart_count": 290, "start_count": 291}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 259, "h264_decode_error_count": 443, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:40:32+09:00", "last_error_summary": "[h264 @ 0x5642d84b2280] error while decoding MB 79 28, bytestream -6", "last_exit_at": "2026-07-02T06:40:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T06:40:35+09:00", "pid": 45176, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.055, "name": "ffmpeg", "pid": 45176, "rss_mb": 136.055, "state": "S (sleeping)", "threads": 106}, "restart_count": 290, "restart_count_per_hour": 26, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 291}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1608.831, "sample_count": 255, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 370666, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:40:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.586, "last_frame_pushed_at": "2026-07-02T06:40:33+09:00", "last_frame_read_age_sec": 5.556, "last_frame_read_at": "2026-07-02T06:40:32+09:00", "mjpeg": {"age_sec": 4.567, "exists": true, "mtime": "2026-07-02T06:40:33.250831+09:00", "mtime_epoch": 1782942033.2508307, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204900}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:40:37+09:00", "process": {"exists": true, "fd_count": 123, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5534.852, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 44.285, "recorders": [{"age_sec": 44.285, "frame_count": 90, "is_recording": true, "is_stopping": false, "pid": 45116, "process": {"exists": true, "fd_count": 5, "hwm_mb": 187.137, "name": "ffmpeg", "pid": 45116, "rss_mb": 187.137, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 10, "returncode": null, "running": true, "started_at": "2026-07-02T06:39:53+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_063953_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [45116], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 25, "pushed_frames": 25, "read_frames": 24, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.857, "pushed": 0.857, "read": 0.822}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 76.929, "count": 24, "max_ms": 83.021, "p95_ms": 82.0}, "gstreamer_push": {"avg_ms": 0.894, "count": 25, "max_ms": 1.003, "p95_ms": 0.964}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.321, "count": 24, "max_ms": 60.349, "p95_ms": 59.913}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 87.066, "count": 24, "max_ms": 106.492, "p95_ms": 100.148}, "yolo11_640_fire_11class": {"avg_ms": 51.447, "count": 24, "max_ms": 70.043, "p95_ms": 69.762}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.201, "count": 24, "max_ms": 100.137, "p95_ms": 97.918}}, "truncated_count": 0}, "processing_total": {"avg_ms": 1015.515, "count": 25, "max_ms": 1198.998, "p95_ms": 1080.033}, "read_interval": {"avg_ms": 1027.318, "count": 24, "max_ms": 1208.485, "p95_ms": 1089.966}, "triton_wait": {"avg_ms": 87.08, "count": 24, "max_ms": 106.506, "p95_ms": 100.159}, "visualize": {"avg_ms": 17.354, "count": 24, "max_ms": 21.409, "p95_ms": 20.537}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.182, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 233, "tracking_summary": 3091}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9530.223, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T06:40:32+09:00", "last_error_summary": "[h264 @ 0x5642d84b2280] error while decoding MB 79 28, bytestream -6", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 259, "h264_decode_error_count": 443, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:40:32+09:00", "last_error_summary": "[h264 @ 0x5642d84b2280] error while decoding MB 79 28, bytestream -6", "last_exit_at": "2026-07-02T06:43:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 201.762, "last_started_at": "2026-07-02T06:40:35+09:00", "pid": 45176, "process": {"exists": false, "pid": 45176}, "restart_count": 290, "restart_count_per_hour": 25, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 291}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 994.651, "sample_count": 431, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 370993, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:43:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T06:43:57+09:00", "last_frame_read_age_sec": 0.496, "last_frame_read_at": "2026-07-02T06:43:57+09:00", "mjpeg": {"age_sec": 0.485, "exists": true, "mtime": "2026-07-02T06:43:57.093868+09:00", "mtime_epoch": 1782942237.093868, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206521}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:43:57+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5534.852, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 54, "pushed_frames": 54, "read_frames": 53, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.826, "pushed": 1.826, "read": 1.792}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 40.845, "count": 53, "max_ms": 48.489, "p95_ms": 46.111}, "gstreamer_push": {"avg_ms": 0.869, "count": 54, "max_ms": 0.939, "p95_ms": 0.914}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.304, "count": 53, "max_ms": 72.042, "p95_ms": 61.512}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.683, "count": 53, "max_ms": 106.892, "p95_ms": 101.042}, "yolo11_640_fire_11class": {"avg_ms": 52.883, "count": 53, "max_ms": 74.269, "p95_ms": 68.025}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.364, "count": 53, "max_ms": 93.734, "p95_ms": 88.776}}, "truncated_count": 0}, "processing_total": {"avg_ms": 551.509, "count": 54, "max_ms": 649.035, "p95_ms": 611.487}, "read_interval": {"avg_ms": 560.278, "count": 53, "max_ms": 658.722, "p95_ms": 620.051}, "triton_wait": {"avg_ms": 81.695, "count": 53, "max_ms": 106.904, "p95_ms": 101.055}, "visualize": {"avg_ms": 17.307, "count": 53, "max_ms": 22.547, "p95_ms": 21.496}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.581, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 233, "tracking_summary": 3091}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9530.215, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 259, "h264_decode_error_count": 443, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:40:32+09:00", "last_error_summary": "[h264 @ 0x5642d84b2280] error while decoding MB 79 28, bytestream -6", "last_exit_at": "2026-07-02T06:43:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 201.764, "last_started_at": "2026-07-02T06:40:35+09:00", "pid": 45176, "process": {"exists": false, "pid": 45176}, "restart_count": 290, "restart_count_per_hour": 25, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 291}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 994.651, "sample_count": 431, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 370993, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:43:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T06:43:57+09:00", "last_frame_read_age_sec": 0.499, "last_frame_read_at": "2026-07-02T06:43:57+09:00", "mjpeg": {"age_sec": 0.488, "exists": true, "mtime": "2026-07-02T06:43:57.093868+09:00", "mtime_epoch": 1782942237.093868, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206521}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:43:57+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5534.852, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 54, "pushed_frames": 54, "read_frames": 53, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.825, "pushed": 1.825, "read": 1.792}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 40.845, "count": 53, "max_ms": 48.489, "p95_ms": 46.111}, "gstreamer_push": {"avg_ms": 0.869, "count": 54, "max_ms": 0.939, "p95_ms": 0.914}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.304, "count": 53, "max_ms": 72.042, "p95_ms": 61.512}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.683, "count": 53, "max_ms": 106.892, "p95_ms": 101.042}, "yolo11_640_fire_11class": {"avg_ms": 52.883, "count": 53, "max_ms": 74.269, "p95_ms": 68.025}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.364, "count": 53, "max_ms": 93.734, "p95_ms": 88.776}}, "truncated_count": 0}, "processing_total": {"avg_ms": 551.509, "count": 54, "max_ms": 649.035, "p95_ms": 611.487}, "read_interval": {"avg_ms": 560.278, "count": 53, "max_ms": 658.722, "p95_ms": 620.051}, "triton_wait": {"avg_ms": 81.695, "count": 53, "max_ms": 106.904, "p95_ms": 101.055}, "visualize": {"avg_ms": 17.307, "count": 53, "max_ms": 22.547, "p95_ms": 21.496}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.584, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 233, "tracking_summary": 3091}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9638.93, "limit_mb": 8796093022207.996}, "detail": {"pid": 45403, "restart_count": 291, "start_count": 292}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 259, "h264_decode_error_count": 443, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:40:32+09:00", "last_error_summary": "[h264 @ 0x5642d84b2280] error while decoding MB 79 28, bytestream -6", "last_exit_at": "2026-07-02T06:44:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T06:44:00+09:00", "pid": 45403, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.098, "name": "ffmpeg", "pid": 45403, "rss_mb": 139.098, "state": "S (sleeping)", "threads": 106}, "restart_count": 291, "restart_count_per_hour": 26, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 292}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 987.051, "sample_count": 426, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 370993, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:43:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.561, "last_frame_pushed_at": "2026-07-02T06:43:57+09:00", "last_frame_read_age_sec": 5.055, "last_frame_read_at": "2026-07-02T06:43:57+09:00", "mjpeg": {"age_sec": 4.543, "exists": true, "mtime": "2026-07-02T06:43:57.593876+09:00", "mtime_epoch": 1782942237.5938756, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206509}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:44:02+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5534.852, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 46, "pushed_frames": 46, "read_frames": 45, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.55, "pushed": 1.55, "read": 1.516}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 40.625, "count": 45, "max_ms": 47.255, "p95_ms": 46.02}, "gstreamer_push": {"avg_ms": 0.867, "count": 46, "max_ms": 0.939, "p95_ms": 0.902}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.772, "count": 45, "max_ms": 72.042, "p95_ms": 61.512}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.925, "count": 45, "max_ms": 106.892, "p95_ms": 101.042}, "yolo11_640_fire_11class": {"avg_ms": 53.249, "count": 45, "max_ms": 74.269, "p95_ms": 68.025}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.928, "count": 45, "max_ms": 93.734, "p95_ms": 88.776}}, "truncated_count": 0}, "processing_total": {"avg_ms": 549.602, "count": 46, "max_ms": 644.742, "p95_ms": 609.515}, "read_interval": {"avg_ms": 558.25, "count": 45, "max_ms": 650.922, "p95_ms": 617.9}, "triton_wait": {"avg_ms": 81.938, "count": 45, "max_ms": 106.904, "p95_ms": 101.055}, "visualize": {"avg_ms": 17.617, "count": 45, "max_ms": 22.547, "p95_ms": 21.496}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.679, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 242, "tracking_summary": 3094}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9704.219, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5605df7f3b00] error while decoding MB 69 20, bytestream -22", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 260, "h264_decode_error_count": 444, "h264_decode_error_count_per_hour": 22, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:46:14+09:00", "last_error_summary": "[h264 @ 0x5605df7f3b00] error while decoding MB 69 20, bytestream -22", "last_exit_at": "2026-07-02T06:44:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 134.3, "last_started_at": "2026-07-02T06:44:00+09:00", "pid": 45403, "process": {"exists": false, "pid": 45403}, "restart_count": 291, "restart_count_per_hour": 25, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 292}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 655.896, "sample_count": 622, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 371339, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:46:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-02T06:46:14+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-02T06:46:14+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-02T06:46:14.391993+09:00", "mtime_epoch": 1782942374.3919933, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 210977}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:46:14+09:00", "process": {"exists": true, "fd_count": 123, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5535.109, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 7.263, "recorders": [{"age_sec": 7.263, "frame_count": 64, "is_recording": true, "is_stopping": false, "pid": 45570, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.688, "name": "ffmpeg", "pid": 45570, "rss_mb": 186.688, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 36, "returncode": null, "running": true, "started_at": "2026-07-02T06:46:07+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_064607_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [45570], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 65, "pushed_frames": 65, "read_frames": 65, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.172, "pushed": 2.172, "read": 2.172}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.417, "count": 64, "max_ms": 37.491, "p95_ms": 36.808}, "gstreamer_push": {"avg_ms": 0.88, "count": 65, "max_ms": 0.99, "p95_ms": 0.929}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.497, "count": 64, "max_ms": 80.476, "p95_ms": 69.45}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.285, "count": 64, "max_ms": 102.985, "p95_ms": 96.759}, "yolo11_640_fire_11class": {"avg_ms": 51.739, "count": 64, "max_ms": 77.924, "p95_ms": 67.371}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.31, "count": 64, "max_ms": 91.25, "p95_ms": 84.653}}, "truncated_count": 0}, "processing_total": {"avg_ms": 456.275, "count": 65, "max_ms": 1162.189, "p95_ms": 497.184}, "read_interval": {"avg_ms": 465.946, "count": 65, "max_ms": 1172.373, "p95_ms": 505.862}, "triton_wait": {"avg_ms": 78.298, "count": 64, "max_ms": 102.998, "p95_ms": 96.772}, "visualize": {"avg_ms": 16.315, "count": 64, "max_ms": 18.438, "p95_ms": 18.168}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.93, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 242, "tracking_summary": 3094}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9703.215, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T06:46:14+09:00", "last_error_summary": "[h264 @ 0x5605df7f3b00] error while decoding MB 69 20, bytestream -22", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 260, "h264_decode_error_count": 444, "h264_decode_error_count_per_hour": 22, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:46:14+09:00", "last_error_summary": "[h264 @ 0x5605df7f3b00] error while decoding MB 69 20, bytestream -22", "last_exit_at": "2026-07-02T06:46:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 134.703, "last_started_at": "2026-07-02T06:44:00+09:00", "pid": 45403, "process": {"exists": false, "pid": 45403}, "restart_count": 291, "restart_count_per_hour": 25, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 292}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 655.896, "sample_count": 623, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 371340, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:46:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T06:46:14+09:00", "last_frame_read_age_sec": 0.454, "last_frame_read_at": "2026-07-02T06:46:14+09:00", "mjpeg": {"age_sec": 0.447, "exists": true, "mtime": "2026-07-02T06:46:14.391993+09:00", "mtime_epoch": 1782942374.3919933, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 210977}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:46:14+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5535.109, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 7.664, "recorders": [{"age_sec": 7.664, "frame_count": 65, "is_recording": true, "is_stopping": false, "pid": 45570, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.688, "name": "ffmpeg", "pid": 45570, "rss_mb": 186.688, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 35, "returncode": null, "running": true, "started_at": "2026-07-02T06:46:07+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_064607_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [45570], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 65, "pushed_frames": 65, "read_frames": 64, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.174, "pushed": 2.174, "read": 2.141}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.42, "count": 64, "max_ms": 37.491, "p95_ms": 36.808}, "gstreamer_push": {"avg_ms": 0.879, "count": 65, "max_ms": 0.99, "p95_ms": 0.929}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.375, "count": 64, "max_ms": 80.476, "p95_ms": 69.45}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.344, "count": 64, "max_ms": 102.985, "p95_ms": 96.759}, "yolo11_640_fire_11class": {"avg_ms": 51.602, "count": 64, "max_ms": 77.924, "p95_ms": 67.371}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.197, "count": 64, "max_ms": 91.25, "p95_ms": 84.653}}, "truncated_count": 0}, "processing_total": {"avg_ms": 456.893, "count": 65, "max_ms": 1162.189, "p95_ms": 497.184}, "read_interval": {"avg_ms": 466.69, "count": 64, "max_ms": 1172.373, "p95_ms": 505.862}, "triton_wait": {"avg_ms": 78.358, "count": 64, "max_ms": 102.998, "p95_ms": 96.772}, "visualize": {"avg_ms": 16.33, "count": 64, "max_ms": 18.438, "p95_ms": 18.168}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.895, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 242, "tracking_summary": 3094}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9703.184, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 260, "h264_decode_error_count": 444, "h264_decode_error_count_per_hour": 22, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:46:14+09:00", "last_error_summary": "[h264 @ 0x5605df7f3b00] error while decoding MB 69 20, bytestream -22", "last_exit_at": "2026-07-02T06:46:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 134.706, "last_started_at": "2026-07-02T06:44:00+09:00", "pid": 45403, "process": {"exists": false, "pid": 45403}, "restart_count": 291, "restart_count_per_hour": 25, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 292}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 655.896, "sample_count": 623, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 371340, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:46:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T06:46:14+09:00", "last_frame_read_age_sec": 0.457, "last_frame_read_at": "2026-07-02T06:46:14+09:00", "mjpeg": {"age_sec": 0.45, "exists": true, "mtime": "2026-07-02T06:46:14.391993+09:00", "mtime_epoch": 1782942374.3919933, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 210977}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:46:14+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5535.109, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 7.667, "recorders": [{"age_sec": 7.667, "frame_count": 65, "is_recording": true, "is_stopping": false, "pid": 45570, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.688, "name": "ffmpeg", "pid": 45570, "rss_mb": 186.688, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 35, "returncode": null, "running": true, "started_at": "2026-07-02T06:46:07+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_064607_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [45570], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 65, "pushed_frames": 65, "read_frames": 64, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.174, "pushed": 2.174, "read": 2.141}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.42, "count": 64, "max_ms": 37.491, "p95_ms": 36.808}, "gstreamer_push": {"avg_ms": 0.879, "count": 65, "max_ms": 0.99, "p95_ms": 0.929}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.375, "count": 64, "max_ms": 80.476, "p95_ms": 69.45}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.344, "count": 64, "max_ms": 102.985, "p95_ms": 96.759}, "yolo11_640_fire_11class": {"avg_ms": 51.602, "count": 64, "max_ms": 77.924, "p95_ms": 67.371}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.197, "count": 64, "max_ms": 91.25, "p95_ms": 84.653}}, "truncated_count": 0}, "processing_total": {"avg_ms": 456.893, "count": 65, "max_ms": 1162.189, "p95_ms": 497.184}, "read_interval": {"avg_ms": 466.69, "count": 64, "max_ms": 1172.373, "p95_ms": 505.862}, "triton_wait": {"avg_ms": 78.358, "count": 64, "max_ms": 102.998, "p95_ms": 96.772}, "visualize": {"avg_ms": 16.33, "count": 64, "max_ms": 18.438, "p95_ms": 18.168}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.898, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 242, "tracking_summary": 3094}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9811.312, "limit_mb": 8796093022207.996}, "detail": {"pid": 45630, "restart_count": 292, "start_count": 293}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 260, "h264_decode_error_count": 444, "h264_decode_error_count_per_hour": 22, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:46:14+09:00", "last_error_summary": "[h264 @ 0x5605df7f3b00] error while decoding MB 69 20, bytestream -22", "last_exit_at": "2026-07-02T06:46:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T06:46:17+09:00", "pid": 45630, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.867, "name": "ffmpeg", "pid": 45630, "rss_mb": 138.867, "state": "S (sleeping)", "threads": 106}, "restart_count": 292, "restart_count_per_hour": 25, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 293}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 649.448, "sample_count": 615, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 371340, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:46:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.966, "last_frame_pushed_at": "2026-07-02T06:46:14+09:00", "last_frame_read_age_sec": 5.416, "last_frame_read_at": "2026-07-02T06:46:14+09:00", "mjpeg": {"age_sec": 4.949, "exists": true, "mtime": "2026-07-02T06:46:14.852000+09:00", "mtime_epoch": 1782942374.8520005, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 211075}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:46:19+09:00", "process": {"exists": true, "fd_count": 123, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5535.109, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": 12.627, "recorders": [{"age_sec": 12.627, "frame_count": 65, "is_recording": true, "is_stopping": false, "pid": 45570, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.688, "name": "ffmpeg", "pid": 45570, "rss_mb": 186.688, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 35, "returncode": null, "running": true, "started_at": "2026-07-02T06:46:07+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_064607_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [45570], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 53, "pushed_frames": 53, "read_frames": 52, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.791, "pushed": 1.791, "read": 1.757}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 33.298, "count": 53, "max_ms": 37.491, "p95_ms": 36.856}, "gstreamer_push": {"avg_ms": 0.881, "count": 53, "max_ms": 0.99, "p95_ms": 0.932}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.256, "count": 53, "max_ms": 80.476, "p95_ms": 69.45}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.062, "count": 53, "max_ms": 102.985, "p95_ms": 99.986}, "yolo11_640_fire_11class": {"avg_ms": 51.714, "count": 53, "max_ms": 77.924, "p95_ms": 69.903}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.965, "count": 53, "max_ms": 91.25, "p95_ms": 86.457}}, "truncated_count": 0}, "processing_total": {"avg_ms": 462.989, "count": 53, "max_ms": 1162.189, "p95_ms": 521.163}, "read_interval": {"avg_ms": 472.95, "count": 52, "max_ms": 1172.373, "p95_ms": 531.981}, "triton_wait": {"avg_ms": 79.075, "count": 53, "max_ms": 102.998, "p95_ms": 99.999}, "visualize": {"avg_ms": 16.468, "count": 53, "max_ms": 18.377, "p95_ms": 18.168}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.593, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 246, "tracking_summary": 3097}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9415.887, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55e981e492c0] error while decoding MB 39 43, bytestream -19", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 261, "h264_decode_error_count": 445, "h264_decode_error_count_per_hour": 22, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:48:30+09:00", "last_error_summary": "[h264 @ 0x55e981e492c0] error while decoding MB 39 43, bytestream -19", "last_exit_at": "2026-07-02T06:46:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.542, "last_started_at": "2026-07-02T06:46:17+09:00", "pid": 45630, "process": {"exists": false, "pid": 45630}, "restart_count": 292, "restart_count_per_hour": 24, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 293}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 546.018, "sample_count": 704, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 371647, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:48:29+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-02T06:48:30+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-02T06:48:30+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-02T06:48:30.298143+09:00", "mtime_epoch": 1782942510.298143, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 213044}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:48:30+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5535.367, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 66, "pushed_frames": 66, "read_frames": 66, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.234, "pushed": 2.234, "read": 2.234}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.301, "count": 66, "max_ms": 36.341, "p95_ms": 35.032}, "gstreamer_push": {"avg_ms": 0.881, "count": 66, "max_ms": 1.015, "p95_ms": 0.953}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.739, "count": 66, "max_ms": 71.856, "p95_ms": 68.007}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.118, "count": 66, "max_ms": 109.022, "p95_ms": 103.55}, "yolo11_640_fire_11class": {"avg_ms": 49.806, "count": 66, "max_ms": 68.442, "p95_ms": 64.872}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.872, "count": 66, "max_ms": 106.592, "p95_ms": 100.204}}, "truncated_count": 0}, "processing_total": {"avg_ms": 444.943, "count": 66, "max_ms": 520.008, "p95_ms": 482.617}, "read_interval": {"avg_ms": 454.504, "count": 66, "max_ms": 530.773, "p95_ms": 492.765}, "triton_wait": {"avg_ms": 79.497, "count": 66, "max_ms": 109.034, "p95_ms": 103.564}, "visualize": {"avg_ms": 17.817, "count": 66, "max_ms": 23.91, "p95_ms": 19.926}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.547, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 246, "tracking_summary": 3097}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9414.836, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T06:48:30+09:00", "last_error_summary": "[h264 @ 0x55e981e492c0] error while decoding MB 39 43, bytestream -19", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 261, "h264_decode_error_count": 445, "h264_decode_error_count_per_hour": 22, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:48:30+09:00", "last_error_summary": "[h264 @ 0x55e981e492c0] error while decoding MB 39 43, bytestream -19", "last_exit_at": "2026-07-02T06:48:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.963, "last_started_at": "2026-07-02T06:46:17+09:00", "pid": 45630, "process": {"exists": false, "pid": 45630}, "restart_count": 292, "restart_count_per_hour": 24, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 293}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 545.491, "sample_count": 704, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 371648, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:48:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T06:48:30+09:00", "last_frame_read_age_sec": 0.475, "last_frame_read_at": "2026-07-02T06:48:30+09:00", "mjpeg": {"age_sec": 0.465, "exists": true, "mtime": "2026-07-02T06:48:30.298143+09:00", "mtime_epoch": 1782942510.298143, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 213044}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:48:30+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5535.367, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 67, "pushed_frames": 67, "read_frames": 66, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.236, "pushed": 2.236, "read": 2.202}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.281, "count": 66, "max_ms": 36.341, "p95_ms": 35.032}, "gstreamer_push": {"avg_ms": 0.881, "count": 67, "max_ms": 1.015, "p95_ms": 0.953}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.488, "count": 66, "max_ms": 71.856, "p95_ms": 68.007}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.952, "count": 66, "max_ms": 109.022, "p95_ms": 103.55}, "yolo11_640_fire_11class": {"avg_ms": 49.795, "count": 66, "max_ms": 68.442, "p95_ms": 64.872}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.63, "count": 66, "max_ms": 106.592, "p95_ms": 100.204}}, "truncated_count": 0}, "processing_total": {"avg_ms": 445.326, "count": 67, "max_ms": 520.008, "p95_ms": 482.617}, "read_interval": {"avg_ms": 454.504, "count": 66, "max_ms": 530.773, "p95_ms": 492.765}, "triton_wait": {"avg_ms": 79.317, "count": 66, "max_ms": 109.034, "p95_ms": 103.564}, "visualize": {"avg_ms": 17.784, "count": 66, "max_ms": 23.91, "p95_ms": 19.587}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.969, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 246, "tracking_summary": 3097}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9414.906, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 261, "h264_decode_error_count": 445, "h264_decode_error_count_per_hour": 22, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:48:30+09:00", "last_error_summary": "[h264 @ 0x55e981e492c0] error while decoding MB 39 43, bytestream -19", "last_exit_at": "2026-07-02T06:48:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.966, "last_started_at": "2026-07-02T06:46:17+09:00", "pid": 45630, "process": {"exists": false, "pid": 45630}, "restart_count": 292, "restart_count_per_hour": 24, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 293}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 545.491, "sample_count": 704, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 371648, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:48:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T06:48:30+09:00", "last_frame_read_age_sec": 0.478, "last_frame_read_at": "2026-07-02T06:48:30+09:00", "mjpeg": {"age_sec": 0.468, "exists": true, "mtime": "2026-07-02T06:48:30.298143+09:00", "mtime_epoch": 1782942510.298143, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 213044}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:48:30+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5535.367, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 67, "pushed_frames": 67, "read_frames": 66, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.235, "pushed": 2.235, "read": 2.202}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.281, "count": 66, "max_ms": 36.341, "p95_ms": 35.032}, "gstreamer_push": {"avg_ms": 0.881, "count": 67, "max_ms": 1.015, "p95_ms": 0.953}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.488, "count": 66, "max_ms": 71.856, "p95_ms": 68.007}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.952, "count": 66, "max_ms": 109.022, "p95_ms": 103.55}, "yolo11_640_fire_11class": {"avg_ms": 49.795, "count": 66, "max_ms": 68.442, "p95_ms": 64.872}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.63, "count": 66, "max_ms": 106.592, "p95_ms": 100.204}}, "truncated_count": 0}, "processing_total": {"avg_ms": 445.326, "count": 67, "max_ms": 520.008, "p95_ms": 482.617}, "read_interval": {"avg_ms": 454.504, "count": 66, "max_ms": 530.773, "p95_ms": 492.765}, "triton_wait": {"avg_ms": 79.317, "count": 66, "max_ms": 109.034, "p95_ms": 103.564}, "visualize": {"avg_ms": 17.784, "count": 66, "max_ms": 23.91, "p95_ms": 19.587}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.972, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 246, "tracking_summary": 3097}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9523.586, "limit_mb": 8796093022207.996}, "detail": {"pid": 45797, "restart_count": 293, "start_count": 294}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 261, "h264_decode_error_count": 445, "h264_decode_error_count_per_hour": 22, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:48:30+09:00", "last_error_summary": "[h264 @ 0x55e981e492c0] error while decoding MB 39 43, bytestream -19", "last_exit_at": "2026-07-02T06:48:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T06:48:33+09:00", "pid": 45797, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.766, "name": "ffmpeg", "pid": 45797, "rss_mb": 138.766, "state": "S (sleeping)", "threads": 106}, "restart_count": 293, "restart_count_per_hour": 25, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 294}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 530.592, "sample_count": 695, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 371648, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:48:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.624, "last_frame_pushed_at": "2026-07-02T06:48:30+09:00", "last_frame_read_age_sec": 5.096, "last_frame_read_at": "2026-07-02T06:48:30+09:00", "mjpeg": {"age_sec": 4.607, "exists": true, "mtime": "2026-07-02T06:48:30.778151+09:00", "mtime_epoch": 1782942510.7781506, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 213696}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:48:35+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5535.367, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 56, "pushed_frames": 56, "read_frames": 55, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.882, "pushed": 1.882, "read": 1.848}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.628, "count": 55, "max_ms": 36.341, "p95_ms": 35.884}, "gstreamer_push": {"avg_ms": 0.879, "count": 56, "max_ms": 1.003, "p95_ms": 0.953}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.795, "count": 55, "max_ms": 71.856, "p95_ms": 68.279}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.787, "count": 55, "max_ms": 109.022, "p95_ms": 103.745}, "yolo11_640_fire_11class": {"avg_ms": 50.221, "count": 55, "max_ms": 68.442, "p95_ms": 65.684}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.272, "count": 55, "max_ms": 106.592, "p95_ms": 103.216}}, "truncated_count": 0}, "processing_total": {"avg_ms": 446.745, "count": 56, "max_ms": 520.008, "p95_ms": 482.617}, "read_interval": {"avg_ms": 455.897, "count": 55, "max_ms": 530.773, "p95_ms": 492.765}, "triton_wait": {"avg_ms": 79.183, "count": 55, "max_ms": 109.034, "p95_ms": 103.877}, "visualize": {"avg_ms": 17.817, "count": 55, "max_ms": 23.91, "p95_ms": 19.587}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.759, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 3, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 251, "tracking_summary": 3099}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9429.395, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55a3c8a3f380] error while decoding MB 30 37, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 262, "h264_decode_error_count": 446, "h264_decode_error_count_per_hour": 22, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:50:50+09:00", "last_error_summary": "[h264 @ 0x55a3c8a3f380] error while decoding MB 30 37, bytestream -9", "last_exit_at": "2026-07-02T06:48:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 137.15, "last_started_at": "2026-07-02T06:48:33+09:00", "pid": 45797, "process": {"exists": false, "pid": 45797}, "restart_count": 293, "restart_count_per_hour": 24, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 294}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 503.081, "sample_count": 650, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 371938, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:50:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-02T06:50:50+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-02T06:50:50+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-02T06:50:50.488397+09:00", "mtime_epoch": 1782942650.4883971, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 207060}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:50:50+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5535.367, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 64, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.154, "pushed": 2.154, "read": 2.154}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.566, "count": 63, "max_ms": 36.76, "p95_ms": 34.377}, "gstreamer_push": {"avg_ms": 0.886, "count": 64, "max_ms": 0.952, "p95_ms": 0.927}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.824, "count": 63, "max_ms": 66.855, "p95_ms": 62.009}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.957, "count": 63, "max_ms": 105.182, "p95_ms": 99.892}, "yolo11_640_fire_11class": {"avg_ms": 52.026, "count": 63, "max_ms": 68.085, "p95_ms": 67.252}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.098, "count": 63, "max_ms": 96.526, "p95_ms": 91.097}}, "truncated_count": 0}, "processing_total": {"avg_ms": 457.432, "count": 64, "max_ms": 580.305, "p95_ms": 514.991}, "read_interval": {"avg_ms": 470.117, "count": 64, "max_ms": 649.804, "p95_ms": 536.997}, "triton_wait": {"avg_ms": 78.97, "count": 63, "max_ms": 105.195, "p95_ms": 99.904}, "visualize": {"avg_ms": 21.774, "count": 63, "max_ms": 29.195, "p95_ms": 25.844}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.708, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 3, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 251, "tracking_summary": 3099}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9428.051, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T06:50:50+09:00", "last_error_summary": "[h264 @ 0x55a3c8a3f380] error while decoding MB 30 37, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 262, "h264_decode_error_count": 446, "h264_decode_error_count_per_hour": 22, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:50:50+09:00", "last_error_summary": "[h264 @ 0x55a3c8a3f380] error while decoding MB 30 37, bytestream -9", "last_exit_at": "2026-07-02T06:50:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 137.464, "last_started_at": "2026-07-02T06:48:33+09:00", "pid": 45797, "process": {"exists": false, "pid": 45797}, "restart_count": 293, "restart_count_per_hour": 24, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 294}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 503.081, "sample_count": 650, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 371939, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:50:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.002, "last_frame_pushed_at": "2026-07-02T06:50:50+09:00", "last_frame_read_age_sec": 0.365, "last_frame_read_at": "2026-07-02T06:50:50+09:00", "mjpeg": {"age_sec": 0.359, "exists": true, "mtime": "2026-07-02T06:50:50.488397+09:00", "mtime_epoch": 1782942650.4883971, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 207060}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:50:50+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5535.367, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 63, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.164, "pushed": 2.164, "read": 2.131}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.497, "count": 64, "max_ms": 36.76, "p95_ms": 34.377}, "gstreamer_push": {"avg_ms": 0.887, "count": 64, "max_ms": 0.952, "p95_ms": 0.927}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.673, "count": 64, "max_ms": 66.855, "p95_ms": 62.009}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.948, "count": 64, "max_ms": 105.182, "p95_ms": 99.892}, "yolo11_640_fire_11class": {"avg_ms": 51.925, "count": 64, "max_ms": 68.085, "p95_ms": 67.252}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.095, "count": 64, "max_ms": 96.526, "p95_ms": 91.097}}, "truncated_count": 0}, "processing_total": {"avg_ms": 456.336, "count": 64, "max_ms": 580.305, "p95_ms": 514.991}, "read_interval": {"avg_ms": 470.567, "count": 63, "max_ms": 649.804, "p95_ms": 536.997}, "triton_wait": {"avg_ms": 78.961, "count": 64, "max_ms": 105.195, "p95_ms": 99.904}, "visualize": {"avg_ms": 21.687, "count": 64, "max_ms": 29.195, "p95_ms": 25.844}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.57, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 3, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 251, "tracking_summary": 3099}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9428.043, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 262, "h264_decode_error_count": 446, "h264_decode_error_count_per_hour": 22, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:50:50+09:00", "last_error_summary": "[h264 @ 0x55a3c8a3f380] error while decoding MB 30 37, bytestream -9", "last_exit_at": "2026-07-02T06:50:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 137.467, "last_started_at": "2026-07-02T06:48:33+09:00", "pid": 45797, "process": {"exists": false, "pid": 45797}, "restart_count": 293, "restart_count_per_hour": 24, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 294}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 503.081, "sample_count": 650, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 371939, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:50:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T06:50:50+09:00", "last_frame_read_age_sec": 0.368, "last_frame_read_at": "2026-07-02T06:50:50+09:00", "mjpeg": {"age_sec": 0.362, "exists": true, "mtime": "2026-07-02T06:50:50.488397+09:00", "mtime_epoch": 1782942650.4883971, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 207060}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:50:50+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5535.367, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 63, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.164, "pushed": 2.164, "read": 2.13}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.497, "count": 64, "max_ms": 36.76, "p95_ms": 34.377}, "gstreamer_push": {"avg_ms": 0.887, "count": 64, "max_ms": 0.952, "p95_ms": 0.927}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.673, "count": 64, "max_ms": 66.855, "p95_ms": 62.009}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.948, "count": 64, "max_ms": 105.182, "p95_ms": 99.892}, "yolo11_640_fire_11class": {"avg_ms": 51.925, "count": 64, "max_ms": 68.085, "p95_ms": 67.252}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.095, "count": 64, "max_ms": 96.526, "p95_ms": 91.097}}, "truncated_count": 0}, "processing_total": {"avg_ms": 456.336, "count": 64, "max_ms": 580.305, "p95_ms": 514.991}, "read_interval": {"avg_ms": 470.567, "count": 63, "max_ms": 649.804, "p95_ms": 536.997}, "triton_wait": {"avg_ms": 78.961, "count": 64, "max_ms": 105.195, "p95_ms": 99.904}, "visualize": {"avg_ms": 21.687, "count": 64, "max_ms": 29.195, "p95_ms": 25.844}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.573, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 3, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 251, "tracking_summary": 3099}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9537.293, "limit_mb": 8796093022207.996}, "detail": {"pid": 45964, "restart_count": 294, "start_count": 295}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 262, "h264_decode_error_count": 446, "h264_decode_error_count_per_hour": 22, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:50:50+09:00", "last_error_summary": "[h264 @ 0x55a3c8a3f380] error while decoding MB 30 37, bytestream -9", "last_exit_at": "2026-07-02T06:50:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T06:50:52+09:00", "pid": 45964, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.75, "name": "ffmpeg", "pid": 45964, "rss_mb": 138.75, "state": "S (sleeping)", "threads": 106}, "restart_count": 294, "restart_count_per_hour": 25, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 295}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 503.081, "sample_count": 640, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 371939, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:50:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.056, "last_frame_pushed_at": "2026-07-02T06:50:50+09:00", "last_frame_read_age_sec": 4.419, "last_frame_read_at": "2026-07-02T06:50:50+09:00", "mjpeg": {"age_sec": 4.04, "exists": true, "mtime": "2026-07-02T06:50:50.860403+09:00", "mtime_epoch": 1782942650.860403, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 207461}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:50:54+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5535.367, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 56, "pushed_frames": 56, "read_frames": 55, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.867, "pushed": 1.867, "read": 1.834}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 31.459, "count": 55, "max_ms": 36.76, "p95_ms": 34.643}, "gstreamer_push": {"avg_ms": 0.887, "count": 56, "max_ms": 0.952, "p95_ms": 0.947}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.631, "count": 55, "max_ms": 66.855, "p95_ms": 62.128}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.7, "count": 55, "max_ms": 105.182, "p95_ms": 101.718}, "yolo11_640_fire_11class": {"avg_ms": 52.582, "count": 55, "max_ms": 68.085, "p95_ms": 67.483}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.329, "count": 55, "max_ms": 96.526, "p95_ms": 91.218}}, "truncated_count": 0}, "processing_total": {"avg_ms": 461.161, "count": 56, "max_ms": 580.305, "p95_ms": 526.42}, "read_interval": {"avg_ms": 472.338, "count": 55, "max_ms": 590.911, "p95_ms": 536.997}, "triton_wait": {"avg_ms": 80.713, "count": 55, "max_ms": 105.195, "p95_ms": 101.731}, "visualize": {"avg_ms": 21.289, "count": 55, "max_ms": 29.195, "p95_ms": 26.106}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.99, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 252, "tracking_summary": 3101}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9434.508, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T06:50:50+09:00", "last_error_summary": "[h264 @ 0x55a3c8a3f380] error while decoding MB 30 37, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 262, "h264_decode_error_count": 446, "h264_decode_error_count_per_hour": 21, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:50:50+09:00", "last_error_summary": "[h264 @ 0x55a3c8a3f380] error while decoding MB 30 37, bytestream -9", "last_exit_at": "2026-07-02T06:53:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.297, "last_started_at": "2026-07-02T06:50:52+09:00", "pid": 45964, "process": {"exists": false, "pid": 45964}, "restart_count": 294, "restart_count_per_hour": 24, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 295}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 497.945, "sample_count": 654, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 372236, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:53:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T06:53:01+09:00", "last_frame_read_age_sec": 0.386, "last_frame_read_at": "2026-07-02T06:53:00+09:00", "mjpeg": {"age_sec": 0.373, "exists": true, "mtime": "2026-07-02T06:53:00.823592+09:00", "mtime_epoch": 1782942780.823592, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 208546}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:53:01+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5535.625, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 77, "pushed_frames": 77, "read_frames": 76, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.571, "pushed": 2.571, "read": 2.538}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.437, "count": 76, "max_ms": 39.871, "p95_ms": 38.5}, "gstreamer_push": {"avg_ms": 0.861, "count": 77, "max_ms": 0.929, "p95_ms": 0.902}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.641, "count": 76, "max_ms": 74.349, "p95_ms": 69.326}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.221, "count": 76, "max_ms": 103.62, "p95_ms": 98.319}, "yolo11_640_fire_11class": {"avg_ms": 55.596, "count": 76, "max_ms": 77.06, "p95_ms": 73.82}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.648, "count": 76, "max_ms": 102.86, "p95_ms": 96.247}}, "truncated_count": 0}, "processing_total": {"avg_ms": 384.591, "count": 77, "max_ms": 440.493, "p95_ms": 421.838}, "read_interval": {"avg_ms": 394.098, "count": 76, "max_ms": 449.795, "p95_ms": 431.638}, "triton_wait": {"avg_ms": 78.289, "count": 76, "max_ms": 103.63, "p95_ms": 98.331}, "visualize": {"avg_ms": 16.429, "count": 76, "max_ms": 18.885, "p95_ms": 17.794}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.946, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 252, "tracking_summary": 3101}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9434.5, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 262, "h264_decode_error_count": 446, "h264_decode_error_count_per_hour": 21, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:50:50+09:00", "last_error_summary": "[h264 @ 0x55a3c8a3f380] error while decoding MB 30 37, bytestream -9", "last_exit_at": "2026-07-02T06:53:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.3, "last_started_at": "2026-07-02T06:50:52+09:00", "pid": 45964, "process": {"exists": false, "pid": 45964}, "restart_count": 294, "restart_count_per_hour": 24, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 295}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 497.945, "sample_count": 654, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 372236, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:53:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T06:53:01+09:00", "last_frame_read_age_sec": 0.389, "last_frame_read_at": "2026-07-02T06:53:00+09:00", "mjpeg": {"age_sec": 0.376, "exists": true, "mtime": "2026-07-02T06:53:00.823592+09:00", "mtime_epoch": 1782942780.823592, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 208546}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:53:01+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5535.625, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 77, "pushed_frames": 77, "read_frames": 76, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.571, "pushed": 2.571, "read": 2.538}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.437, "count": 76, "max_ms": 39.871, "p95_ms": 38.5}, "gstreamer_push": {"avg_ms": 0.861, "count": 77, "max_ms": 0.929, "p95_ms": 0.902}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.641, "count": 76, "max_ms": 74.349, "p95_ms": 69.326}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.221, "count": 76, "max_ms": 103.62, "p95_ms": 98.319}, "yolo11_640_fire_11class": {"avg_ms": 55.596, "count": 76, "max_ms": 77.06, "p95_ms": 73.82}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.648, "count": 76, "max_ms": 102.86, "p95_ms": 96.247}}, "truncated_count": 0}, "processing_total": {"avg_ms": 384.591, "count": 77, "max_ms": 440.493, "p95_ms": 421.838}, "read_interval": {"avg_ms": 394.098, "count": 76, "max_ms": 449.795, "p95_ms": 431.638}, "triton_wait": {"avg_ms": 78.289, "count": 76, "max_ms": 103.63, "p95_ms": 98.331}, "visualize": {"avg_ms": 16.429, "count": 76, "max_ms": 18.885, "p95_ms": 17.794}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.949, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 252, "tracking_summary": 3101}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9539.613, "limit_mb": 8796093022207.996}, "detail": {"pid": 46071, "restart_count": 295, "start_count": 296}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 262, "h264_decode_error_count": 446, "h264_decode_error_count_per_hour": 21, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:50:50+09:00", "last_error_summary": "[h264 @ 0x55a3c8a3f380] error while decoding MB 30 37, bytestream -9", "last_exit_at": "2026-07-02T06:53:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T06:53:03+09:00", "pid": 46071, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.949, "name": "ffmpeg", "pid": 46071, "rss_mb": 138.949, "state": "S (sleeping)", "threads": 106}, "restart_count": 295, "restart_count_per_hour": 25, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 296}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 498.315, "sample_count": 644, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 372236, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:53:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.054, "last_frame_pushed_at": "2026-07-02T06:53:01+09:00", "last_frame_read_age_sec": 4.438, "last_frame_read_at": "2026-07-02T06:53:00+09:00", "mjpeg": {"age_sec": 4.036, "exists": true, "mtime": "2026-07-02T06:53:01.211601+09:00", "mtime_epoch": 1782942781.2116015, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 207949}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:53:05+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5535.625, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 66, "pushed_frames": 66, "read_frames": 65, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.218, "pushed": 2.218, "read": 2.185}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 32.609, "count": 65, "max_ms": 39.871, "p95_ms": 38.5}, "gstreamer_push": {"avg_ms": 0.86, "count": 66, "max_ms": 0.929, "p95_ms": 0.9}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.792, "count": 65, "max_ms": 71.261, "p95_ms": 68.533}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.903, "count": 65, "max_ms": 99.261, "p95_ms": 96.714}, "yolo11_640_fire_11class": {"avg_ms": 55.69, "count": 65, "max_ms": 75.088, "p95_ms": 73.444}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.343, "count": 65, "max_ms": 97.447, "p95_ms": 94.468}}, "truncated_count": 0}, "processing_total": {"avg_ms": 386.021, "count": 66, "max_ms": 440.493, "p95_ms": 421.838}, "read_interval": {"avg_ms": 395.579, "count": 65, "max_ms": 449.795, "p95_ms": 431.638}, "triton_wait": {"avg_ms": 77.915, "count": 65, "max_ms": 99.275, "p95_ms": 96.725}, "visualize": {"avg_ms": 16.375, "count": 65, "max_ms": 18.885, "p95_ms": 17.664}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.75, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 253, "tracking_summary": 3105}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9442.488, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55d68ae655c0] error while decoding MB 74 26, bytestream -13", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 263, "h264_decode_error_count": 447, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:54:51+09:00", "last_error_summary": "[h264 @ 0x55d68ae655c0] error while decoding MB 74 26, bytestream -13", "last_exit_at": "2026-07-02T06:53:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.488, "last_started_at": "2026-07-02T06:53:03+09:00", "pid": 46071, "process": {"exists": false, "pid": 46071}, "restart_count": 295, "restart_count_per_hour": 24, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 296}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 478.867, "sample_count": 845, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 372658, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:54:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-02T06:54:51+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-02T06:54:51+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-02T06:54:51.694283+09:00", "mtime_epoch": 1782942891.6942828, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200718}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:54:51+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5535.883, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 130, "pushed_frames": 130, "read_frames": 130, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.347, "pushed": 4.347, "read": 4.347}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 20.813, "count": 129, "max_ms": 28.167, "p95_ms": 23.007}, "gstreamer_push": {"avg_ms": 0.875, "count": 130, "max_ms": 1.258, "p95_ms": 0.932}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.806, "count": 129, "max_ms": 71.21, "p95_ms": 69.062}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.022, "count": 129, "max_ms": 98.763, "p95_ms": 95.227}, "yolo11_640_fire_11class": {"avg_ms": 57.363, "count": 129, "max_ms": 77.396, "p95_ms": 72.846}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.149, "count": 129, "max_ms": 97.912, "p95_ms": 94.097}}, "truncated_count": 0}, "processing_total": {"avg_ms": 221.858, "count": 130, "max_ms": 284.713, "p95_ms": 267.239}, "read_interval": {"avg_ms": 231.444, "count": 130, "max_ms": 295.141, "p95_ms": 277.461}, "triton_wait": {"avg_ms": 76.117, "count": 129, "max_ms": 98.774, "p95_ms": 95.575}, "visualize": {"avg_ms": 8.617, "count": 129, "max_ms": 11.754, "p95_ms": 10.344}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.908, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 253, "tracking_summary": 3105}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9441.121, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T06:54:51+09:00", "last_error_summary": "[h264 @ 0x55d68ae655c0] error while decoding MB 74 26, bytestream -13", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 263, "h264_decode_error_count": 447, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:54:51+09:00", "last_error_summary": "[h264 @ 0x55d68ae655c0] error while decoding MB 74 26, bytestream -13", "last_exit_at": "2026-07-02T06:54:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.684, "last_started_at": "2026-07-02T06:53:03+09:00", "pid": 46071, "process": {"exists": false, "pid": 46071}, "restart_count": 295, "restart_count_per_hour": 24, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 296}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 478.867, "sample_count": 846, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 372659, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:54:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T06:54:51+09:00", "last_frame_read_age_sec": 0.244, "last_frame_read_at": "2026-07-02T06:54:51+09:00", "mjpeg": {"age_sec": 0.236, "exists": true, "mtime": "2026-07-02T06:54:51.694283+09:00", "mtime_epoch": 1782942891.6942828, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200718}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:54:51+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5535.883, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 130, "pushed_frames": 130, "read_frames": 129, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.352, "pushed": 4.352, "read": 4.318}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 20.811, "count": 130, "max_ms": 28.167, "p95_ms": 23.007}, "gstreamer_push": {"avg_ms": 0.875, "count": 130, "max_ms": 1.258, "p95_ms": 0.932}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.906, "count": 130, "max_ms": 71.21, "p95_ms": 69.062}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.151, "count": 130, "max_ms": 98.763, "p95_ms": 95.227}, "yolo11_640_fire_11class": {"avg_ms": 57.438, "count": 130, "max_ms": 77.396, "p95_ms": 72.846}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.295, "count": 130, "max_ms": 97.912, "p95_ms": 94.097}}, "truncated_count": 0}, "processing_total": {"avg_ms": 221.945, "count": 130, "max_ms": 284.713, "p95_ms": 267.239}, "read_interval": {"avg_ms": 231.387, "count": 129, "max_ms": 295.141, "p95_ms": 277.461}, "triton_wait": {"avg_ms": 76.246, "count": 130, "max_ms": 98.774, "p95_ms": 95.575}, "visualize": {"avg_ms": 8.616, "count": 130, "max_ms": 11.754, "p95_ms": 10.344}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.872, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 253, "tracking_summary": 3105}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9441.113, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 263, "h264_decode_error_count": 447, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:54:51+09:00", "last_error_summary": "[h264 @ 0x55d68ae655c0] error while decoding MB 74 26, bytestream -13", "last_exit_at": "2026-07-02T06:54:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 108.687, "last_started_at": "2026-07-02T06:53:03+09:00", "pid": 46071, "process": {"exists": false, "pid": 46071}, "restart_count": 295, "restart_count_per_hour": 24, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 296}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 478.867, "sample_count": 846, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 372659, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:54:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T06:54:51+09:00", "last_frame_read_age_sec": 0.248, "last_frame_read_at": "2026-07-02T06:54:51+09:00", "mjpeg": {"age_sec": 0.24, "exists": true, "mtime": "2026-07-02T06:54:51.694283+09:00", "mtime_epoch": 1782942891.6942828, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200718}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:54:51+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5535.883, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 130, "pushed_frames": 130, "read_frames": 129, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.351, "pushed": 4.351, "read": 4.318}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 20.811, "count": 130, "max_ms": 28.167, "p95_ms": 23.007}, "gstreamer_push": {"avg_ms": 0.875, "count": 130, "max_ms": 1.258, "p95_ms": 0.932}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.906, "count": 130, "max_ms": 71.21, "p95_ms": 69.062}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.151, "count": 130, "max_ms": 98.763, "p95_ms": 95.227}, "yolo11_640_fire_11class": {"avg_ms": 57.438, "count": 130, "max_ms": 77.396, "p95_ms": 72.846}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.295, "count": 130, "max_ms": 97.912, "p95_ms": 94.097}}, "truncated_count": 0}, "processing_total": {"avg_ms": 221.945, "count": 130, "max_ms": 284.713, "p95_ms": 267.239}, "read_interval": {"avg_ms": 231.387, "count": 129, "max_ms": 295.141, "p95_ms": 277.461}, "triton_wait": {"avg_ms": 76.246, "count": 130, "max_ms": 98.774, "p95_ms": 95.575}, "visualize": {"avg_ms": 8.616, "count": 130, "max_ms": 11.754, "p95_ms": 10.344}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.876, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 253, "tracking_summary": 3105}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9550.141, "limit_mb": 8796093022207.996}, "detail": {"pid": 46238, "restart_count": 296, "start_count": 297}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 263, "h264_decode_error_count": 447, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:54:51+09:00", "last_error_summary": "[h264 @ 0x55d68ae655c0] error while decoding MB 74 26, bytestream -13", "last_exit_at": "2026-07-02T06:54:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T06:54:54+09:00", "pid": 46238, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.289, "name": "ffmpeg", "pid": 46238, "rss_mb": 139.289, "state": "S (sleeping)", "threads": 106}, "restart_count": 296, "restart_count_per_hour": 25, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 297}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 478.684, "sample_count": 835, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 372659, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:54:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.887, "last_frame_pushed_at": "2026-07-02T06:54:51+09:00", "last_frame_read_age_sec": 5.129, "last_frame_read_at": "2026-07-02T06:54:51+09:00", "mjpeg": {"age_sec": 4.869, "exists": true, "mtime": "2026-07-02T06:54:51.946289+09:00", "mtime_epoch": 1782942891.9462888, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199800}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T06:54:56+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5535.883, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 110, "pushed_frames": 110, "read_frames": 109, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.69, "pushed": 3.69, "read": 3.657}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 20.719, "count": 110, "max_ms": 28.167, "p95_ms": 23.138}, "gstreamer_push": {"avg_ms": 0.872, "count": 110, "max_ms": 0.967, "p95_ms": 0.932}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.048, "count": 110, "max_ms": 71.21, "p95_ms": 69.17}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.152, "count": 110, "max_ms": 98.763, "p95_ms": 95.227}, "yolo11_640_fire_11class": {"avg_ms": 57.395, "count": 110, "max_ms": 77.396, "p95_ms": 72.949}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.163, "count": 110, "max_ms": 97.912, "p95_ms": 94.097}}, "truncated_count": 0}, "processing_total": {"avg_ms": 219.092, "count": 110, "max_ms": 284.713, "p95_ms": 261.922}, "read_interval": {"avg_ms": 228.473, "count": 109, "max_ms": 295.141, "p95_ms": 271.123}, "triton_wait": {"avg_ms": 76.248, "count": 110, "max_ms": 98.774, "p95_ms": 95.575}, "visualize": {"avg_ms": 8.484, "count": 110, "max_ms": 11.754, "p95_ms": 9.815}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.808, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 9, "tracking_summary": 4536}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9394.18, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x555b9ffff180] error while decoding MB 52 29, bytestream -45 [in#0/rtsp @ 0x555b9f32fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555b9f32fe00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 68, "h264_decode_error_count": 68, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:59:33+09:00", "last_error_summary": "[h264 @ 0x555b9ffff180] error while decoding MB 52 29, bytestream -45 [in#0/rtsp @ 0x555b9f32fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555b9f32fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T04:32:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 8820.123, "last_started_at": "2026-07-02T04:32:33+09:00", "pid": 37984, "process": {"exists": false, "pid": 37984}, "restart_count": 67, "restart_count_per_hour": 0, "rtsp_timeout_count": 124, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 68}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 150.138, "sample_count": 2326, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 544347, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:59:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.078, "last_frame_pushed_at": "2026-07-02T06:59:33+09:00", "last_frame_read_age_sec": 0.066, "last_frame_read_at": "2026-07-02T06:59:33+09:00", "mjpeg": {"age_sec": 0.062, "exists": true, "mtime": "2026-07-02T06:59:33.584613+09:00", "mtime_epoch": 1782943173.5846128, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 253393}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T06:59:33+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1384.668, "name": "celery", "pid": 267, "rss_mb": 1126.289, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 96, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.203, "pushed": 3.203, "read": 3.203}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.2, "count": 96, "max_ms": 7.016, "p95_ms": 4.427}, "gstreamer_push": {"avg_ms": 1.544, "count": 96, "max_ms": 3.884, "p95_ms": 3.649}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.466, "count": 96, "max_ms": 74.467, "p95_ms": 73.365}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 86.814, "count": 96, "max_ms": 98.928, "p95_ms": 93.377}, "yolo11_640_fire_11class": {"avg_ms": 74.263, "count": 96, "max_ms": 85.52, "p95_ms": 83.604}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.895, "count": 96, "max_ms": 88.734, "p95_ms": 86.347}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.84, "count": 96, "max_ms": 111.81, "p95_ms": 106.791}, "read_interval": {"avg_ms": 312.573, "count": 96, "max_ms": 18991.307, "p95_ms": 126.626}, "triton_wait": {"avg_ms": 86.826, "count": 96, "max_ms": 98.941, "p95_ms": 93.39}, "visualize": {"avg_ms": 3.69, "count": 96, "max_ms": 4.69, "p95_ms": 4.191}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.97, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 9, "tracking_summary": 4536}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9393.859, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T06:59:33+09:00", "last_error_summary": "[h264 @ 0x555b9ffff180] error while decoding MB 52 29, bytestream -45 [in#0/rtsp @ 0x555b9f32fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555b9f32fe00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 68, "h264_decode_error_count": 68, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:59:33+09:00", "last_error_summary": "[h264 @ 0x555b9ffff180] error while decoding MB 52 29, bytestream -45 [in#0/rtsp @ 0x555b9f32fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555b9f32fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T06:59:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 8820.154, "last_started_at": "2026-07-02T04:32:33+09:00", "pid": 37984, "process": {"exists": false, "pid": 37984}, "restart_count": 67, "restart_count_per_hour": 0, "rtsp_timeout_count": 124, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 68}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 150.138, "sample_count": 2327, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 544348, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:59:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T06:59:33+09:00", "last_frame_read_age_sec": 0.098, "last_frame_read_at": "2026-07-02T06:59:33+09:00", "mjpeg": {"age_sec": 0.093, "exists": true, "mtime": "2026-07-02T06:59:33.584613+09:00", "mtime_epoch": 1782943173.5846128, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 253393}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T06:59:33+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1384.668, "name": "celery", "pid": 267, "rss_mb": 1126.289, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 97, "read_frames": 96, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.2, "pushed": 3.233, "read": 3.2}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.238, "count": 96, "max_ms": 7.778, "p95_ms": 4.584}, "gstreamer_push": {"avg_ms": 1.54, "count": 97, "max_ms": 3.884, "p95_ms": 3.649}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.422, "count": 96, "max_ms": 74.467, "p95_ms": 73.365}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 86.634, "count": 96, "max_ms": 98.928, "p95_ms": 93.377}, "yolo11_640_fire_11class": {"avg_ms": 74.114, "count": 96, "max_ms": 85.52, "p95_ms": 83.604}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.767, "count": 96, "max_ms": 88.734, "p95_ms": 86.347}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.699, "count": 96, "max_ms": 111.81, "p95_ms": 106.791}, "read_interval": {"avg_ms": 312.573, "count": 96, "max_ms": 18991.307, "p95_ms": 126.626}, "triton_wait": {"avg_ms": 86.647, "count": 96, "max_ms": 98.941, "p95_ms": 93.39}, "visualize": {"avg_ms": 3.694, "count": 96, "max_ms": 4.69, "p95_ms": 4.191}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 9, "tracking_summary": 4536}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9393.773, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 68, "h264_decode_error_count": 68, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:59:33+09:00", "last_error_summary": "[h264 @ 0x555b9ffff180] error while decoding MB 52 29, bytestream -45 [in#0/rtsp @ 0x555b9f32fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555b9f32fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T06:59:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 8820.161, "last_started_at": "2026-07-02T04:32:33+09:00", "pid": 37984, "process": {"exists": false, "pid": 37984}, "restart_count": 67, "restart_count_per_hour": 0, "rtsp_timeout_count": 124, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 68}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 150.138, "sample_count": 2327, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 544348, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:59:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-02T06:59:33+09:00", "last_frame_read_age_sec": 0.105, "last_frame_read_at": "2026-07-02T06:59:33+09:00", "mjpeg": {"age_sec": 0.101, "exists": true, "mtime": "2026-07-02T06:59:33.584613+09:00", "mtime_epoch": 1782943173.5846128, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 253393}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T06:59:33+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1384.668, "name": "celery", "pid": 267, "rss_mb": 1126.289, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 96, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.201, "pushed": 3.201, "read": 3.201}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.238, "count": 96, "max_ms": 7.778, "p95_ms": 4.584}, "gstreamer_push": {"avg_ms": 1.542, "count": 96, "max_ms": 3.884, "p95_ms": 3.649}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.422, "count": 96, "max_ms": 74.467, "p95_ms": 73.365}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 86.634, "count": 96, "max_ms": 98.928, "p95_ms": 93.377}, "yolo11_640_fire_11class": {"avg_ms": 74.114, "count": 96, "max_ms": 85.52, "p95_ms": 83.604}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.767, "count": 96, "max_ms": 88.734, "p95_ms": 86.347}}, "truncated_count": 0}, "processing_total": {"avg_ms": 99.699, "count": 96, "max_ms": 111.81, "p95_ms": 106.791}, "read_interval": {"avg_ms": 312.573, "count": 96, "max_ms": 18991.307, "p95_ms": 126.626}, "triton_wait": {"avg_ms": 86.647, "count": 96, "max_ms": 98.941, "p95_ms": 93.39}, "visualize": {"avg_ms": 3.694, "count": 96, "max_ms": 4.69, "p95_ms": 4.191}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 9, "tracking_summary": 4536}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9414.758, "limit_mb": 8796093022207.996}, "detail": {"pid": 46345, "restart_count": 68, "start_count": 69}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 68, "h264_decode_error_count": 68, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T06:59:33+09:00", "last_error_summary": "[h264 @ 0x555b9ffff180] error while decoding MB 52 29, bytestream -45 [in#0/rtsp @ 0x555b9f32fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x555b9f32fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T06:59:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T06:59:35+09:00", "pid": 46345, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.227, "name": "ffmpeg", "pid": 46345, "rss_mb": 58.227, "state": "S (sleeping)", "threads": 1}, "restart_count": 68, "restart_count_per_hour": 1, "rtsp_timeout_count": 124, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 69}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 150.499, "sample_count": 2283, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 544348, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T06:59:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.051, "last_frame_pushed_at": "2026-07-02T06:59:33+09:00", "last_frame_read_age_sec": 4.143, "last_frame_read_at": "2026-07-02T06:59:33+09:00", "mjpeg": {"age_sec": 4.034, "exists": true, "mtime": "2026-07-02T06:59:33.688615+09:00", "mtime_epoch": 1782943173.688615, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 246883}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T06:59:37+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1384.668, "name": "celery", "pid": 267, "rss_mb": 1126.289, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 61, "pushed_frames": 61, "read_frames": 60, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.039, "pushed": 2.039, "read": 2.005}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.267, "count": 61, "max_ms": 7.778, "p95_ms": 4.584}, "gstreamer_push": {"avg_ms": 1.536, "count": 61, "max_ms": 3.884, "p95_ms": 3.466}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.771, "count": 61, "max_ms": 74.467, "p95_ms": 73.43}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 87.406, "count": 61, "max_ms": 98.928, "p95_ms": 93.377}, "yolo11_640_fire_11class": {"avg_ms": 74.2, "count": 61, "max_ms": 85.52, "p95_ms": 82.757}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.46, "count": 61, "max_ms": 88.734, "p95_ms": 86.762}}, "truncated_count": 0}, "processing_total": {"avg_ms": 100.484, "count": 61, "max_ms": 111.81, "p95_ms": 106.791}, "read_interval": {"avg_ms": 431.311, "count": 60, "max_ms": 18991.307, "p95_ms": 127.461}, "triton_wait": {"avg_ms": 87.419, "count": 61, "max_ms": 98.941, "p95_ms": 93.39}, "visualize": {"avg_ms": 3.687, "count": 61, "max_ms": 4.69, "p95_ms": 4.011}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.918, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 253, "tracking_summary": 3130}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9444.754, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x564f6b987ac0] error while decoding MB 79 9, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 264, "h264_decode_error_count": 448, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:00:50+09:00", "last_error_summary": "[h264 @ 0x564f6b987ac0] error while decoding MB 79 9, bytestream -7", "last_exit_at": "2026-07-02T06:54:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 355.52, "last_started_at": "2026-07-02T06:54:54+09:00", "pid": 46238, "process": {"exists": false, "pid": 46238}, "restart_count": 296, "restart_count_per_hour": 23, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 297}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.148, "sample_count": 2568, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 375705, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:00:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.068, "last_frame_pushed_at": "2026-07-02T07:00:50+09:00", "last_frame_read_age_sec": 0.056, "last_frame_read_at": "2026-07-02T07:00:50+09:00", "mjpeg": {"age_sec": 0.052, "exists": true, "mtime": "2026-07-02T07:00:50.282241+09:00", "mtime_epoch": 1782943250.2822413, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 201676}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:00:50+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5537.172, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 251, "pushed_frames": 251, "read_frames": 251, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.38, "pushed": 8.38, "read": 8.38}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.4, "count": 251, "max_ms": 22.188, "p95_ms": 15.213}, "gstreamer_push": {"avg_ms": 1.281, "count": 251, "max_ms": 1.632, "p95_ms": 1.361}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.294, "count": 251, "max_ms": 70.163, "p95_ms": 65.91}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.909, "count": 251, "max_ms": 94.62, "p95_ms": 88.372}, "yolo11_640_fire_11class": {"avg_ms": 68.465, "count": 251, "max_ms": 81.752, "p95_ms": 74.479}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.212, "count": 251, "max_ms": 83.858, "p95_ms": 81.043}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.767, "count": 251, "max_ms": 117.151, "p95_ms": 110.849}, "read_interval": {"avg_ms": 119.538, "count": 251, "max_ms": 132.113, "p95_ms": 126.868}, "triton_wait": {"avg_ms": 80.922, "count": 251, "max_ms": 94.633, "p95_ms": 88.383}, "visualize": {"avg_ms": 3.555, "count": 251, "max_ms": 5.297, "p95_ms": 3.849}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.954, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 253, "tracking_summary": 3130}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9446.762, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T07:00:50+09:00", "last_error_summary": "[h264 @ 0x564f6b987ac0] error while decoding MB 79 9, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 264, "h264_decode_error_count": 448, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:00:50+09:00", "last_error_summary": "[h264 @ 0x564f6b987ac0] error while decoding MB 79 9, bytestream -7", "last_exit_at": "2026-07-02T07:00:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 355.564, "last_started_at": "2026-07-02T06:54:54+09:00", "pid": 46238, "process": {"exists": false, "pid": 46238}, "restart_count": 296, "restart_count_per_hour": 23, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 297}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.148, "sample_count": 2569, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 375706, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:00:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T07:00:50+09:00", "last_frame_read_age_sec": 0.1, "last_frame_read_at": "2026-07-02T07:00:50+09:00", "mjpeg": {"age_sec": 0.096, "exists": true, "mtime": "2026-07-02T07:00:50.282241+09:00", "mtime_epoch": 1782943250.2822413, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 201676}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:00:50+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5537.172, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 252, "pushed_frames": 252, "read_frames": 251, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.4, "pushed": 8.4, "read": 8.367}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.399, "count": 251, "max_ms": 22.188, "p95_ms": 15.213}, "gstreamer_push": {"avg_ms": 1.281, "count": 252, "max_ms": 1.632, "p95_ms": 1.361}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.248, "count": 251, "max_ms": 70.163, "p95_ms": 65.91}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.852, "count": 251, "max_ms": 94.62, "p95_ms": 88.372}, "yolo11_640_fire_11class": {"avg_ms": 68.429, "count": 251, "max_ms": 81.752, "p95_ms": 74.479}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.16, "count": 251, "max_ms": 83.858, "p95_ms": 81.043}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.717, "count": 252, "max_ms": 117.151, "p95_ms": 110.849}, "read_interval": {"avg_ms": 119.538, "count": 251, "max_ms": 132.113, "p95_ms": 126.868}, "triton_wait": {"avg_ms": 80.864, "count": 251, "max_ms": 94.633, "p95_ms": 88.383}, "visualize": {"avg_ms": 3.555, "count": 251, "max_ms": 5.297, "p95_ms": 3.849}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 253, "tracking_summary": 3130}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9446.684, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 264, "h264_decode_error_count": 448, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:00:50+09:00", "last_error_summary": "[h264 @ 0x564f6b987ac0] error while decoding MB 79 9, bytestream -7", "last_exit_at": "2026-07-02T07:00:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 355.573, "last_started_at": "2026-07-02T06:54:54+09:00", "pid": 46238, "process": {"exists": false, "pid": 46238}, "restart_count": 296, "restart_count_per_hour": 23, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 297}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.148, "sample_count": 2569, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 375706, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:00:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-02T07:00:50+09:00", "last_frame_read_age_sec": 0.109, "last_frame_read_at": "2026-07-02T07:00:50+09:00", "mjpeg": {"age_sec": 0.105, "exists": true, "mtime": "2026-07-02T07:00:50.282241+09:00", "mtime_epoch": 1782943250.2822413, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 201676}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:00:50+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5537.172, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 251, "pushed_frames": 251, "read_frames": 251, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.37, "pushed": 8.37, "read": 8.37}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.399, "count": 251, "max_ms": 22.188, "p95_ms": 15.213}, "gstreamer_push": {"avg_ms": 1.281, "count": 251, "max_ms": 1.632, "p95_ms": 1.361}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.248, "count": 251, "max_ms": 70.163, "p95_ms": 65.91}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.852, "count": 251, "max_ms": 94.62, "p95_ms": 88.372}, "yolo11_640_fire_11class": {"avg_ms": 68.429, "count": 251, "max_ms": 81.752, "p95_ms": 74.479}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.16, "count": 251, "max_ms": 83.858, "p95_ms": 81.043}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.708, "count": 251, "max_ms": 117.151, "p95_ms": 110.849}, "read_interval": {"avg_ms": 119.538, "count": 251, "max_ms": 132.113, "p95_ms": 126.868}, "triton_wait": {"avg_ms": 80.864, "count": 251, "max_ms": 94.633, "p95_ms": 88.383}, "visualize": {"avg_ms": 3.555, "count": 251, "max_ms": 5.297, "p95_ms": 3.849}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.988, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 253, "tracking_summary": 3130}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9552.375, "limit_mb": 8796093022207.996}, "detail": {"pid": 46497, "restart_count": 297, "start_count": 298}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 264, "h264_decode_error_count": 448, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:00:50+09:00", "last_error_summary": "[h264 @ 0x564f6b987ac0] error while decoding MB 79 9, bytestream -7", "last_exit_at": "2026-07-02T07:00:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T07:00:53+09:00", "pid": 46497, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.93, "name": "ffmpeg", "pid": 46497, "rss_mb": 138.93, "state": "S (sleeping)", "threads": 106}, "restart_count": 297, "restart_count_per_hour": 24, "rtsp_timeout_count": 104, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 298}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.009, "sample_count": 2528, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 375706, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:00:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 5.014, "last_frame_pushed_at": "2026-07-02T07:00:50+09:00", "last_frame_read_age_sec": 5.106, "last_frame_read_at": "2026-07-02T07:00:50+09:00", "mjpeg": {"age_sec": 4.994, "exists": true, "mtime": "2026-07-02T07:00:50.390244+09:00", "mtime_epoch": 1782943250.3902438, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 201729}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:00:55+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5537.172, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 210, "pushed_frames": 210, "read_frames": 210, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.003, "pushed": 7.003, "read": 7.003}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.419, "count": 210, "max_ms": 22.188, "p95_ms": 15.213}, "gstreamer_push": {"avg_ms": 1.283, "count": 210, "max_ms": 1.632, "p95_ms": 1.371}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.744, "count": 210, "max_ms": 70.163, "p95_ms": 66.655}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.302, "count": 210, "max_ms": 94.62, "p95_ms": 88.579}, "yolo11_640_fire_11class": {"avg_ms": 67.895, "count": 210, "max_ms": 81.752, "p95_ms": 74.592}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.578, "count": 210, "max_ms": 83.858, "p95_ms": 81.213}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.195, "count": 210, "max_ms": 117.151, "p95_ms": 110.958}, "read_interval": {"avg_ms": 119.074, "count": 210, "max_ms": 132.113, "p95_ms": 126.891}, "triton_wait": {"avg_ms": 80.315, "count": 210, "max_ms": 94.633, "p95_ms": 88.591}, "visualize": {"avg_ms": 3.562, "count": 210, "max_ms": 5.297, "p95_ms": 3.856}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.988, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 253, "tracking_summary": 3149}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9444.93, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 8, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55ac6f8fde80] cabac decode of qscale diff failed at 58 10 [h264 @ 0x55ac6f8fde80] error while decoding MB 58 10, bytestream 3731 [h264 @ 0x55ac6f80fd80] cabac decode of qscale diff failed at 73 3 [h264 @ 0x55ac6f80fd80] error while decoding MB 73 3, bytestream 64752 [h264 @ 0x55ac6f8fde80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ac6f8fde80] error while decoding MB 0 35, bytestream 4466 [h264 @ 0x55ac6f8af380] left block unavailable for requested intra4x4 mode ", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 265, "h264_decode_error_count": 456, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:05:15+09:00", "last_error_summary": "[h264 @ 0x55ac6f8fde80] cabac decode of qscale diff failed at 58 10 [h264 @ 0x55ac6f8fde80] error while decoding MB 58 10, bytestream 3731 [h264 @ 0x55ac6f80fd80] cabac decode of qscale diff failed at 73 3 [h264 @ 0x55ac6f80fd80] error while decoding MB 73 3, bytestream 64752 [h264 @ 0x55ac6f8fde80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ac6f8fde80] error while decoding MB 0 35, bytestream 4466 [h264 @ 0x55ac6f8af380] left block unavailable for requested intra4x4 mode ", "last_exit_at": "2026-07-02T07:00:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 261.944, "last_started_at": "2026-07-02T07:00:53+09:00", "pid": 46497, "process": {"exists": false, "pid": 46497}, "restart_count": 297, "restart_count_per_hour": 22, "rtsp_timeout_count": 106, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 298}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 118.934, "sample_count": 2476, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 377889, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:05:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.072, "last_frame_pushed_at": "2026-07-02T07:05:15+09:00", "last_frame_read_age_sec": 0.059, "last_frame_read_at": "2026-07-02T07:05:15+09:00", "mjpeg": {"age_sec": 0.052, "exists": true, "mtime": "2026-07-02T07:05:15.275664+09:00", "mtime_epoch": 1782943515.2756639, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 202985}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:05:15+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5538.203, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 100, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.301, "pushed": 3.301, "read": 3.334}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.292, "count": 99, "max_ms": 16.707, "p95_ms": 16.275}, "gstreamer_push": {"avg_ms": 1.305, "count": 99, "max_ms": 1.613, "p95_ms": 1.402}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.217, "count": 99, "max_ms": 69.529, "p95_ms": 58.154}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.331, "count": 99, "max_ms": 99.524, "p95_ms": 97.691}, "yolo11_640_fire_11class": {"avg_ms": 57.573, "count": 99, "max_ms": 80.999, "p95_ms": 76.691}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.8, "count": 99, "max_ms": 90.708, "p95_ms": 89.174}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.161, "count": 99, "max_ms": 122.462, "p95_ms": 120.517}, "read_interval": {"avg_ms": 300.294, "count": 100, "max_ms": 10128.554, "p95_ms": 136.8}, "triton_wait": {"avg_ms": 71.345, "count": 99, "max_ms": 99.537, "p95_ms": 97.704}, "visualize": {"avg_ms": 3.618, "count": 99, "max_ms": 4.505, "p95_ms": 3.858}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 253, "tracking_summary": 3149}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9444.289, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T07:05:15+09:00", "last_error_summary": "[h264 @ 0x55ac6f8fde80] cabac decode of qscale diff failed at 58 10 [h264 @ 0x55ac6f8fde80] error while decoding MB 58 10, bytestream 3731 [h264 @ 0x55ac6f80fd80] cabac decode of qscale diff failed at 73 3 [h264 @ 0x55ac6f80fd80] error while decoding MB 73 3, bytestream 64752 [h264 @ 0x55ac6f8fde80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ac6f8fde80] error while decoding MB 0 35, bytestream 4466 [h264 @ 0x55ac6f8af380] left block unavailable for requested intra4x4 mode ", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 265, "h264_decode_error_count": 456, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:05:15+09:00", "last_error_summary": "[h264 @ 0x55ac6f8fde80] cabac decode of qscale diff failed at 58 10 [h264 @ 0x55ac6f8fde80] error while decoding MB 58 10, bytestream 3731 [h264 @ 0x55ac6f80fd80] cabac decode of qscale diff failed at 73 3 [h264 @ 0x55ac6f80fd80] error while decoding MB 73 3, bytestream 64752 [h264 @ 0x55ac6f8fde80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ac6f8fde80] error while decoding MB 0 35, bytestream 4466 [h264 @ 0x55ac6f8af380] left block unavailable for requested intra4x4 mode ", "last_exit_at": "2026-07-02T07:05:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 261.984, "last_started_at": "2026-07-02T07:00:53+09:00", "pid": 46497, "process": {"exists": false, "pid": 46497}, "restart_count": 297, "restart_count_per_hour": 22, "rtsp_timeout_count": 106, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 298}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 118.934, "sample_count": 2476, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 377890, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:05:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T07:05:15+09:00", "last_frame_read_age_sec": 0.099, "last_frame_read_at": "2026-07-02T07:05:15+09:00", "mjpeg": {"age_sec": 0.092, "exists": true, "mtime": "2026-07-02T07:05:15.275664+09:00", "mtime_epoch": 1782943515.2756639, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 202985}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:05:15+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5538.203, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.339, "pushed": 3.339, "read": 3.306}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.316, "count": 100, "max_ms": 16.707, "p95_ms": 16.275}, "gstreamer_push": {"avg_ms": 1.305, "count": 100, "max_ms": 1.613, "p95_ms": 1.401}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.195, "count": 100, "max_ms": 69.529, "p95_ms": 57.078}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.27, "count": 100, "max_ms": 99.524, "p95_ms": 97.411}, "yolo11_640_fire_11class": {"avg_ms": 57.524, "count": 100, "max_ms": 80.999, "p95_ms": 76.611}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.714, "count": 100, "max_ms": 90.708, "p95_ms": 89.068}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.125, "count": 100, "max_ms": 122.462, "p95_ms": 120.512}, "read_interval": {"avg_ms": 302.355, "count": 99, "max_ms": 10128.554, "p95_ms": 137.645}, "triton_wait": {"avg_ms": 71.283, "count": 100, "max_ms": 99.537, "p95_ms": 97.424}, "visualize": {"avg_ms": 3.62, "count": 100, "max_ms": 4.505, "p95_ms": 3.828}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.948, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 253, "tracking_summary": 3149}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9444.281, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 265, "h264_decode_error_count": 456, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:05:15+09:00", "last_error_summary": "[h264 @ 0x55ac6f8fde80] cabac decode of qscale diff failed at 58 10 [h264 @ 0x55ac6f8fde80] error while decoding MB 58 10, bytestream 3731 [h264 @ 0x55ac6f80fd80] cabac decode of qscale diff failed at 73 3 [h264 @ 0x55ac6f80fd80] error while decoding MB 73 3, bytestream 64752 [h264 @ 0x55ac6f8fde80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ac6f8fde80] error while decoding MB 0 35, bytestream 4466 [h264 @ 0x55ac6f8af380] left block unavailable for requested intra4x4 mode ", "last_exit_at": "2026-07-02T07:05:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 261.992, "last_started_at": "2026-07-02T07:00:53+09:00", "pid": 46497, "process": {"exists": false, "pid": 46497}, "restart_count": 297, "restart_count_per_hour": 22, "rtsp_timeout_count": 106, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 298}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 118.934, "sample_count": 2476, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 377890, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:05:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-02T07:05:15+09:00", "last_frame_read_age_sec": 0.107, "last_frame_read_at": "2026-07-02T07:05:15+09:00", "mjpeg": {"age_sec": 0.1, "exists": true, "mtime": "2026-07-02T07:05:15.275664+09:00", "mtime_epoch": 1782943515.2756639, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 202985}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:05:15+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5538.203, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.338, "pushed": 3.338, "read": 3.305}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.316, "count": 100, "max_ms": 16.707, "p95_ms": 16.275}, "gstreamer_push": {"avg_ms": 1.305, "count": 100, "max_ms": 1.613, "p95_ms": 1.401}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.195, "count": 100, "max_ms": 69.529, "p95_ms": 57.078}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.27, "count": 100, "max_ms": 99.524, "p95_ms": 97.411}, "yolo11_640_fire_11class": {"avg_ms": 57.524, "count": 100, "max_ms": 80.999, "p95_ms": 76.611}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.714, "count": 100, "max_ms": 90.708, "p95_ms": 89.068}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.125, "count": 100, "max_ms": 122.462, "p95_ms": 120.512}, "read_interval": {"avg_ms": 302.355, "count": 99, "max_ms": 10128.554, "p95_ms": 137.645}, "triton_wait": {"avg_ms": 71.283, "count": 100, "max_ms": 99.537, "p95_ms": 97.424}, "visualize": {"avg_ms": 3.62, "count": 100, "max_ms": 4.505, "p95_ms": 3.828}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.957, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 253, "tracking_summary": 3149}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9555.18, "limit_mb": 8796093022207.996}, "detail": {"pid": 46604, "restart_count": 298, "start_count": 299}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 265, "h264_decode_error_count": 456, "h264_decode_error_count_per_hour": 25, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:05:15+09:00", "last_error_summary": "[h264 @ 0x55ac6f8fde80] cabac decode of qscale diff failed at 58 10 [h264 @ 0x55ac6f8fde80] error while decoding MB 58 10, bytestream 3731 [h264 @ 0x55ac6f80fd80] cabac decode of qscale diff failed at 73 3 [h264 @ 0x55ac6f80fd80] error while decoding MB 73 3, bytestream 64752 [h264 @ 0x55ac6f8fde80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ac6f8fde80] error while decoding MB 0 35, bytestream 4466 [h264 @ 0x55ac6f8af380] left block unavailable for requested intra4x4 mode ", "last_exit_at": "2026-07-02T07:05:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T07:05:18+09:00", "pid": 46604, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.113, "name": "ffmpeg", "pid": 46604, "rss_mb": 139.113, "state": "S (sleeping)", "threads": 106}, "restart_count": 298, "restart_count_per_hour": 23, "rtsp_timeout_count": 106, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 299}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 118.946, "sample_count": 2438, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 377890, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:05:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.737, "last_frame_pushed_at": "2026-07-02T07:05:15+09:00", "last_frame_read_age_sec": 4.83, "last_frame_read_at": "2026-07-02T07:05:15+09:00", "mjpeg": {"age_sec": 4.719, "exists": true, "mtime": "2026-07-02T07:05:15.379666+09:00", "mtime_epoch": 1782943515.3796659, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203047}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:05:20+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5538.203, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 57, "pushed_frames": 57, "read_frames": 56, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.903, "pushed": 1.903, "read": 1.869}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.384, "count": 57, "max_ms": 16.707, "p95_ms": 16.549}, "gstreamer_push": {"avg_ms": 1.296, "count": 57, "max_ms": 1.401, "p95_ms": 1.393}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.473, "count": 57, "max_ms": 69.529, "p95_ms": 59.694}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.107, "count": 57, "max_ms": 99.524, "p95_ms": 98.031}, "yolo11_640_fire_11class": {"avg_ms": 57.372, "count": 57, "max_ms": 80.999, "p95_ms": 77.408}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.373, "count": 57, "max_ms": 90.708, "p95_ms": 89.719}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.026, "count": 57, "max_ms": 122.462, "p95_ms": 120.517}, "read_interval": {"avg_ms": 450.311, "count": 56, "max_ms": 10128.554, "p95_ms": 138.336}, "triton_wait": {"avg_ms": 71.121, "count": 57, "max_ms": 99.537, "p95_ms": 98.046}, "visualize": {"avg_ms": 3.63, "count": 57, "max_ms": 4.505, "p95_ms": 3.858}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.96, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 9, "tracking_summary": 4572}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9409.812, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[NULL @ 0x556b7d190200] missing picture in access unit with size 95 [h264 @ 0x556b7d9c5e00] no frame! [vist#0:0/h264 @ 0x556b7d190840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 69, "h264_decode_error_count": 69, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:09:39+09:00", "last_error_summary": "[NULL @ 0x556b7d190200] missing picture in access unit with size 95 [h264 @ 0x556b7d9c5e00] no frame! [vist#0:0/h264 @ 0x556b7d190840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T06:59:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 603.398, "last_started_at": "2026-07-02T06:59:35+09:00", "pid": 46345, "process": {"exists": false, "pid": 46345}, "restart_count": 68, "restart_count_per_hour": 1, "rtsp_timeout_count": 124, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 69}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 171.683, "sample_count": 2250, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 548758, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:09:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.078, "last_frame_pushed_at": "2026-07-02T07:09:39+09:00", "last_frame_read_age_sec": 0.066, "last_frame_read_at": "2026-07-02T07:09:39+09:00", "mjpeg": {"age_sec": 0.063, "exists": true, "mtime": "2026-07-02T07:09:39.056846+09:00", "mtime_epoch": 1782943779.0568461, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 264816}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T07:09:39+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1384.668, "name": "celery", "pid": 267, "rss_mb": 1131.422, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 264, "pushed_frames": 264, "read_frames": 264, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.804, "pushed": 8.804, "read": 8.804}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.951, "count": 264, "max_ms": 11.379, "p95_ms": 6.818}, "gstreamer_push": {"avg_ms": 1.533, "count": 264, "max_ms": 5.672, "p95_ms": 3.654}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.56, "count": 264, "max_ms": 76.532, "p95_ms": 61.294}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.072, "count": 264, "max_ms": 99.693, "p95_ms": 95.827}, "yolo11_640_fire_11class": {"avg_ms": 61.304, "count": 264, "max_ms": 77.304, "p95_ms": 73.96}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.708, "count": 264, "max_ms": 92.895, "p95_ms": 89.186}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.703, "count": 264, "max_ms": 151.087, "p95_ms": 116.409}, "read_interval": {"avg_ms": 113.732, "count": 264, "max_ms": 163.881, "p95_ms": 131.079}, "triton_wait": {"avg_ms": 80.11, "count": 264, "max_ms": 99.706, "p95_ms": 95.841}, "visualize": {"avg_ms": 3.843, "count": 264, "max_ms": 6.838, "p95_ms": 5.074}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.986, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 9, "tracking_summary": 4572}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9409.016, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T07:09:39+09:00", "last_error_summary": "[NULL @ 0x556b7d190200] missing picture in access unit with size 95 [h264 @ 0x556b7d9c5e00] no frame! [vist#0:0/h264 @ 0x556b7d190840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 69, "h264_decode_error_count": 69, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:09:39+09:00", "last_error_summary": "[NULL @ 0x556b7d190200] missing picture in access unit with size 95 [h264 @ 0x556b7d9c5e00] no frame! [vist#0:0/h264 @ 0x556b7d190840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T07:09:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 603.479, "last_started_at": "2026-07-02T06:59:35+09:00", "pid": 46345, "process": {"exists": false, "pid": 46345}, "restart_count": 68, "restart_count_per_hour": 1, "rtsp_timeout_count": 124, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 69}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 171.683, "sample_count": 2251, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 548759, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:09:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T07:09:39+09:00", "last_frame_read_age_sec": 0.147, "last_frame_read_at": "2026-07-02T07:09:39+09:00", "mjpeg": {"age_sec": 0.143, "exists": true, "mtime": "2026-07-02T07:09:39.056846+09:00", "mtime_epoch": 1782943779.0568461, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 264816}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T07:09:39+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1384.668, "name": "celery", "pid": 267, "rss_mb": 1131.422, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 264, "pushed_frames": 264, "read_frames": 263, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.817, "pushed": 8.817, "read": 8.784}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.959, "count": 264, "max_ms": 11.379, "p95_ms": 6.818}, "gstreamer_push": {"avg_ms": 1.532, "count": 264, "max_ms": 5.672, "p95_ms": 3.654}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.628, "count": 264, "max_ms": 76.532, "p95_ms": 61.538}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.057, "count": 264, "max_ms": 99.693, "p95_ms": 95.827}, "yolo11_640_fire_11class": {"avg_ms": 61.3, "count": 264, "max_ms": 77.304, "p95_ms": 73.96}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.693, "count": 264, "max_ms": 92.895, "p95_ms": 89.186}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.83, "count": 264, "max_ms": 151.087, "p95_ms": 120.009}, "read_interval": {"avg_ms": 113.701, "count": 263, "max_ms": 163.881, "p95_ms": 131.079}, "triton_wait": {"avg_ms": 80.094, "count": 264, "max_ms": 99.706, "p95_ms": 95.841}, "visualize": {"avg_ms": 3.848, "count": 264, "max_ms": 6.838, "p95_ms": 5.074}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.942, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 9, "tracking_summary": 4572}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9409.004, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 69, "h264_decode_error_count": 69, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:09:39+09:00", "last_error_summary": "[NULL @ 0x556b7d190200] missing picture in access unit with size 95 [h264 @ 0x556b7d9c5e00] no frame! [vist#0:0/h264 @ 0x556b7d190840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T07:09:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 603.484, "last_started_at": "2026-07-02T06:59:35+09:00", "pid": 46345, "process": {"exists": false, "pid": 46345}, "restart_count": 68, "restart_count_per_hour": 1, "rtsp_timeout_count": 124, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 69}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 171.683, "sample_count": 2250, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 548759, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:09:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-02T07:09:39+09:00", "last_frame_read_age_sec": 0.152, "last_frame_read_at": "2026-07-02T07:09:39+09:00", "mjpeg": {"age_sec": 0.149, "exists": true, "mtime": "2026-07-02T07:09:39.056846+09:00", "mtime_epoch": 1782943779.0568461, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 264816}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T07:09:39+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1384.668, "name": "celery", "pid": 267, "rss_mb": 1131.422, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 264, "pushed_frames": 264, "read_frames": 263, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.815, "pushed": 8.815, "read": 8.782}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.959, "count": 264, "max_ms": 11.379, "p95_ms": 6.818}, "gstreamer_push": {"avg_ms": 1.532, "count": 264, "max_ms": 5.672, "p95_ms": 3.654}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.628, "count": 264, "max_ms": 76.532, "p95_ms": 61.538}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.057, "count": 264, "max_ms": 99.693, "p95_ms": 95.827}, "yolo11_640_fire_11class": {"avg_ms": 61.3, "count": 264, "max_ms": 77.304, "p95_ms": 73.96}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.693, "count": 264, "max_ms": 92.895, "p95_ms": 89.186}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.83, "count": 264, "max_ms": 151.087, "p95_ms": 120.009}, "read_interval": {"avg_ms": 113.701, "count": 263, "max_ms": 163.881, "p95_ms": 131.079}, "triton_wait": {"avg_ms": 80.094, "count": 264, "max_ms": 99.706, "p95_ms": 95.841}, "visualize": {"avg_ms": 3.848, "count": 264, "max_ms": 6.838, "p95_ms": 5.074}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.948, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 9, "tracking_summary": 4572}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9588.609, "limit_mb": 8796093022207.996}, "detail": {"pid": 46771, "restart_count": 69, "start_count": 70}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 69, "h264_decode_error_count": 69, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:09:39+09:00", "last_error_summary": "[NULL @ 0x556b7d190200] missing picture in access unit with size 95 [h264 @ 0x556b7d9c5e00] no frame! [vist#0:0/h264 @ 0x556b7d190840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T07:09:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T07:09:41+09:00", "pid": 46771, "process": {"exists": true, "fd_count": 5, "hwm_mb": 207.414, "name": "ffmpeg", "pid": 46771, "rss_mb": 207.414, "state": "S (sleeping)", "threads": 106}, "restart_count": 69, "restart_count_per_hour": 2, "rtsp_timeout_count": 124, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 70}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 171.729, "sample_count": 2215, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 548759, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:09:39+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.101, "last_frame_pushed_at": "2026-07-02T07:09:39+09:00", "last_frame_read_age_sec": 4.242, "last_frame_read_at": "2026-07-02T07:09:39+09:00", "mjpeg": {"age_sec": 4.083, "exists": true, "mtime": "2026-07-02T07:09:39.212849+09:00", "mtime_epoch": 1782943779.2128491, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 264723}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T07:09:43+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1384.668, "name": "celery", "pid": 267, "rss_mb": 1131.422, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 231, "pushed_frames": 231, "read_frames": 230, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.706, "pushed": 7.706, "read": 7.673}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.994, "count": 231, "max_ms": 11.379, "p95_ms": 6.861}, "gstreamer_push": {"avg_ms": 1.519, "count": 231, "max_ms": 5.672, "p95_ms": 3.678}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.423, "count": 231, "max_ms": 76.532, "p95_ms": 61.603}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.216, "count": 231, "max_ms": 99.693, "p95_ms": 95.866}, "yolo11_640_fire_11class": {"avg_ms": 60.128, "count": 231, "max_ms": 77.304, "p95_ms": 73.804}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.597, "count": 231, "max_ms": 92.895, "p95_ms": 89.644}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.567, "count": 231, "max_ms": 151.087, "p95_ms": 123.257}, "read_interval": {"avg_ms": 112.347, "count": 230, "max_ms": 163.881, "p95_ms": 133.224}, "triton_wait": {"avg_ms": 78.257, "count": 231, "max_ms": 99.706, "p95_ms": 95.878}, "visualize": {"avg_ms": 3.868, "count": 231, "max_ms": 6.838, "p95_ms": 5.074}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.975, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3171}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9466.859, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56274ad6b140] error while decoding MB 57 7, bytestream -6", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 266, "h264_decode_error_count": 457, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:10:41+09:00", "last_error_summary": "[h264 @ 0x56274ad6b140] error while decoding MB 57 7, bytestream -6", "last_exit_at": "2026-07-02T07:05:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 323.838, "last_started_at": "2026-07-02T07:05:18+09:00", "pid": 46604, "process": {"exists": false, "pid": 46604}, "restart_count": 298, "restart_count_per_hour": 20, "rtsp_timeout_count": 106, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 299}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.244, "sample_count": 2515, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 380606, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:10:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.07, "last_frame_pushed_at": "2026-07-02T07:10:41+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-02T07:10:41+09:00", "mjpeg": {"age_sec": 0.053, "exists": true, "mtime": "2026-07-02T07:10:41.882059+09:00", "mtime_epoch": 1782943841.882059, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197140}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:10:41+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5539.492, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 214, "pushed_frames": 214, "read_frames": 214, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.134, "pushed": 7.134, "read": 7.134}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 15.142, "count": 214, "max_ms": 23.031, "p95_ms": 19.458}, "gstreamer_push": {"avg_ms": 1.21, "count": 214, "max_ms": 1.534, "p95_ms": 1.362}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.322, "count": 214, "max_ms": 71.126, "p95_ms": 68.893}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.621, "count": 214, "max_ms": 96.808, "p95_ms": 91.727}, "yolo11_640_fire_11class": {"avg_ms": 61.744, "count": 214, "max_ms": 80.501, "p95_ms": 71.636}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.028, "count": 214, "max_ms": 89.816, "p95_ms": 85.871}}, "truncated_count": 0}, "processing_total": {"avg_ms": 126.106, "count": 214, "max_ms": 819.046, "p95_ms": 196.768}, "read_interval": {"avg_ms": 140.41, "count": 214, "max_ms": 831.796, "p95_ms": 208.07}, "triton_wait": {"avg_ms": 78.634, "count": 214, "max_ms": 96.818, "p95_ms": 91.738}, "visualize": {"avg_ms": 5.042, "count": 214, "max_ms": 10.781, "p95_ms": 10.216}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3171}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9466.586, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T07:10:41+09:00", "last_error_summary": "[h264 @ 0x56274ad6b140] error while decoding MB 57 7, bytestream -6", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 266, "h264_decode_error_count": 457, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:10:41+09:00", "last_error_summary": "[h264 @ 0x56274ad6b140] error while decoding MB 57 7, bytestream -6", "last_exit_at": "2026-07-02T07:10:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 323.879, "last_started_at": "2026-07-02T07:05:18+09:00", "pid": 46604, "process": {"exists": false, "pid": 46604}, "restart_count": 298, "restart_count_per_hour": 20, "rtsp_timeout_count": 106, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 299}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.244, "sample_count": 2516, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 380607, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:10:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T07:10:41+09:00", "last_frame_read_age_sec": 0.095, "last_frame_read_at": "2026-07-02T07:10:41+09:00", "mjpeg": {"age_sec": 0.095, "exists": true, "mtime": "2026-07-02T07:10:41.882059+09:00", "mtime_epoch": 1782943841.882059, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197140}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:10:41+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5539.492, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 214, "pushed_frames": 214, "read_frames": 213, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.154, "pushed": 7.154, "read": 7.121}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 15.151, "count": 214, "max_ms": 23.031, "p95_ms": 19.458}, "gstreamer_push": {"avg_ms": 1.21, "count": 214, "max_ms": 1.534, "p95_ms": 1.362}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.236, "count": 214, "max_ms": 71.126, "p95_ms": 68.893}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.516, "count": 214, "max_ms": 96.808, "p95_ms": 91.727}, "yolo11_640_fire_11class": {"avg_ms": 61.694, "count": 214, "max_ms": 80.501, "p95_ms": 71.636}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.945, "count": 214, "max_ms": 89.816, "p95_ms": 85.871}}, "truncated_count": 0}, "processing_total": {"avg_ms": 126.011, "count": 214, "max_ms": 819.046, "p95_ms": 196.768}, "read_interval": {"avg_ms": 140.494, "count": 213, "max_ms": 831.796, "p95_ms": 208.07}, "triton_wait": {"avg_ms": 78.529, "count": 214, "max_ms": 96.818, "p95_ms": 91.738}, "visualize": {"avg_ms": 5.044, "count": 214, "max_ms": 10.781, "p95_ms": 10.216}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.913, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3171}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9466.703, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 266, "h264_decode_error_count": 457, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:10:41+09:00", "last_error_summary": "[h264 @ 0x56274ad6b140] error while decoding MB 57 7, bytestream -6", "last_exit_at": "2026-07-02T07:10:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 323.888, "last_started_at": "2026-07-02T07:05:18+09:00", "pid": 46604, "process": {"exists": false, "pid": 46604}, "restart_count": 298, "restart_count_per_hour": 20, "rtsp_timeout_count": 106, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 299}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.244, "sample_count": 2516, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 380607, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:10:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-02T07:10:41+09:00", "last_frame_read_age_sec": 0.103, "last_frame_read_at": "2026-07-02T07:10:41+09:00", "mjpeg": {"age_sec": 0.103, "exists": true, "mtime": "2026-07-02T07:10:41.882059+09:00", "mtime_epoch": 1782943841.882059, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197140}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:10:41+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5539.492, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 214, "pushed_frames": 214, "read_frames": 213, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.152, "pushed": 7.152, "read": 7.119}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 15.151, "count": 214, "max_ms": 23.031, "p95_ms": 19.458}, "gstreamer_push": {"avg_ms": 1.21, "count": 214, "max_ms": 1.534, "p95_ms": 1.362}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.236, "count": 214, "max_ms": 71.126, "p95_ms": 68.893}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.516, "count": 214, "max_ms": 96.808, "p95_ms": 91.727}, "yolo11_640_fire_11class": {"avg_ms": 61.694, "count": 214, "max_ms": 80.501, "p95_ms": 71.636}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.945, "count": 214, "max_ms": 89.816, "p95_ms": 85.871}}, "truncated_count": 0}, "processing_total": {"avg_ms": 126.011, "count": 214, "max_ms": 819.046, "p95_ms": 196.768}, "read_interval": {"avg_ms": 140.494, "count": 213, "max_ms": 831.796, "p95_ms": 208.07}, "triton_wait": {"avg_ms": 78.529, "count": 214, "max_ms": 96.818, "p95_ms": 91.738}, "visualize": {"avg_ms": 5.044, "count": 214, "max_ms": 10.781, "p95_ms": 10.216}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.922, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3171}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9568.727, "limit_mb": 8796093022207.996}, "detail": {"pid": 46983, "restart_count": 299, "start_count": 300}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 266, "h264_decode_error_count": 457, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:10:41+09:00", "last_error_summary": "[h264 @ 0x56274ad6b140] error while decoding MB 57 7, bytestream -6", "last_exit_at": "2026-07-02T07:10:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T07:10:44+09:00", "pid": 46983, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.539, "name": "ffmpeg", "pid": 46983, "rss_mb": 136.539, "state": "S (sleeping)", "threads": 106}, "restart_count": 299, "restart_count_per_hour": 21, "rtsp_timeout_count": 106, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 300}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 149.495, "sample_count": 2473, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 380607, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:10:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.756, "last_frame_pushed_at": "2026-07-02T07:10:41+09:00", "last_frame_read_age_sec": 4.844, "last_frame_read_at": "2026-07-02T07:10:41+09:00", "mjpeg": {"age_sec": 4.74, "exists": true, "mtime": "2026-07-02T07:10:41.986061+09:00", "mtime_epoch": 1782943841.986061, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197192}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:10:46+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5539.492, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 174, "pushed_frames": 174, "read_frames": 173, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.803, "pushed": 5.803, "read": 5.769}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 15.487, "count": 174, "max_ms": 23.031, "p95_ms": 19.708}, "gstreamer_push": {"avg_ms": 1.195, "count": 174, "max_ms": 1.534, "p95_ms": 1.359}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.27, "count": 174, "max_ms": 71.126, "p95_ms": 69.18}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.284, "count": 174, "max_ms": 96.808, "p95_ms": 91.732}, "yolo11_640_fire_11class": {"avg_ms": 61.759, "count": 174, "max_ms": 80.501, "p95_ms": 71.86}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.435, "count": 174, "max_ms": 89.816, "p95_ms": 86.778}}, "truncated_count": 0}, "processing_total": {"avg_ms": 131.621, "count": 174, "max_ms": 819.046, "p95_ms": 198.243}, "read_interval": {"avg_ms": 145.821, "count": 173, "max_ms": 831.796, "p95_ms": 209.717}, "triton_wait": {"avg_ms": 78.296, "count": 174, "max_ms": 96.818, "p95_ms": 91.744}, "visualize": {"avg_ms": 5.402, "count": 174, "max_ms": 10.781, "p95_ms": 10.296}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.986, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 9, "tracking_summary": 4582}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9412.148, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55c6cd97c7c0] error while decoding MB 52 41, bytestream -27 [in#0/rtsp @ 0x55c6cd840e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c6cd840e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 70, "h264_decode_error_count": 70, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:12:24+09:00", "last_error_summary": "[h264 @ 0x55c6cd97c7c0] error while decoding MB 52 41, bytestream -27 [in#0/rtsp @ 0x55c6cd840e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c6cd840e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T07:09:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 162.949, "last_started_at": "2026-07-02T07:09:41+09:00", "pid": 46771, "process": {"exists": false, "pid": 46771}, "restart_count": 69, "restart_count_per_hour": 2, "rtsp_timeout_count": 126, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 70}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 110.775, "sample_count": 2371, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 549928, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:12:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.082, "last_frame_pushed_at": "2026-07-02T07:12:24+09:00", "last_frame_read_age_sec": 0.07, "last_frame_read_at": "2026-07-02T07:12:24+09:00", "mjpeg": {"age_sec": 0.068, "exists": true, "mtime": "2026-07-02T07:12:24.176021+09:00", "mtime_epoch": 1782943944.176021, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 258676}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T07:12:24+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1384.668, "name": "celery", "pid": 267, "rss_mb": 1131.043, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 9.375, "pushed": 9.375, "read": 10.156}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.792, "count": 12, "max_ms": 5.316, "p95_ms": 5.316}, "gstreamer_push": {"avg_ms": 1.525, "count": 12, "max_ms": 3.736, "p95_ms": 3.736}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.624, "count": 12, "max_ms": 67.692, "p95_ms": 67.692}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.608, "count": 12, "max_ms": 93.28, "p95_ms": 93.28}, "yolo11_640_fire_11class": {"avg_ms": 60.6, "count": 12, "max_ms": 79.783, "p95_ms": 79.783}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.839, "count": 12, "max_ms": 86.652, "p95_ms": 86.652}}, "truncated_count": 0}, "processing_total": {"avg_ms": 86.22, "count": 12, "max_ms": 106.773, "p95_ms": 106.773}, "read_interval": {"avg_ms": 2351.885, "count": 13, "max_ms": 29365.102, "p95_ms": 29365.102}, "triton_wait": {"avg_ms": 72.62, "count": 12, "max_ms": 93.291, "p95_ms": 93.291}, "visualize": {"avg_ms": 3.726, "count": 12, "max_ms": 4.501, "p95_ms": 4.501}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 1.28, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 9, "tracking_summary": 4582}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9411.961, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T07:12:24+09:00", "last_error_summary": "[h264 @ 0x55c6cd97c7c0] error while decoding MB 52 41, bytestream -27 [in#0/rtsp @ 0x55c6cd840e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c6cd840e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 70, "h264_decode_error_count": 70, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:12:24+09:00", "last_error_summary": "[h264 @ 0x55c6cd97c7c0] error while decoding MB 52 41, bytestream -27 [in#0/rtsp @ 0x55c6cd840e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c6cd840e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T07:12:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 162.992, "last_started_at": "2026-07-02T07:09:41+09:00", "pid": 46771, "process": {"exists": false, "pid": 46771}, "restart_count": 69, "restart_count_per_hour": 2, "rtsp_timeout_count": 126, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 70}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 110.775, "sample_count": 2371, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 549929, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:12:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T07:12:24+09:00", "last_frame_read_age_sec": 0.113, "last_frame_read_at": "2026-07-02T07:12:24+09:00", "mjpeg": {"age_sec": 0.111, "exists": true, "mtime": "2026-07-02T07:12:24.176021+09:00", "mtime_epoch": 1782943944.176021, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 258676}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T07:12:24+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1384.668, "name": "celery", "pid": 267, "rss_mb": 1131.043, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 9.829, "pushed": 9.829, "read": 9.829}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.779, "count": 13, "max_ms": 5.316, "p95_ms": 5.316}, "gstreamer_push": {"avg_ms": 1.511, "count": 13, "max_ms": 3.736, "p95_ms": 3.736}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.423, "count": 13, "max_ms": 67.692, "p95_ms": 67.692}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.099, "count": 13, "max_ms": 93.28, "p95_ms": 93.28}, "yolo11_640_fire_11class": {"avg_ms": 60.988, "count": 13, "max_ms": 79.783, "p95_ms": 79.783}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.074, "count": 13, "max_ms": 86.652, "p95_ms": 86.652}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.686, "count": 13, "max_ms": 106.773, "p95_ms": 106.773}, "read_interval": {"avg_ms": 2351.885, "count": 13, "max_ms": 29365.102, "p95_ms": 29365.102}, "triton_wait": {"avg_ms": 74.111, "count": 13, "max_ms": 93.291, "p95_ms": 93.291}, "visualize": {"avg_ms": 3.718, "count": 13, "max_ms": 4.501, "p95_ms": 4.501}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 1.323, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 9, "tracking_summary": 4582}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9411.781, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 70, "h264_decode_error_count": 70, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:12:24+09:00", "last_error_summary": "[h264 @ 0x55c6cd97c7c0] error while decoding MB 52 41, bytestream -27 [in#0/rtsp @ 0x55c6cd840e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c6cd840e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T07:12:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.0, "last_started_at": "2026-07-02T07:09:41+09:00", "pid": 46771, "process": {"exists": false, "pid": 46771}, "restart_count": 69, "restart_count_per_hour": 2, "rtsp_timeout_count": 126, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 70}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 110.775, "sample_count": 2371, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 549929, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:12:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-02T07:12:24+09:00", "last_frame_read_age_sec": 0.12, "last_frame_read_at": "2026-07-02T07:12:24+09:00", "mjpeg": {"age_sec": 0.118, "exists": true, "mtime": "2026-07-02T07:12:24.176021+09:00", "mtime_epoch": 1782943944.176021, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 258676}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T07:12:24+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1384.668, "name": "celery", "pid": 267, "rss_mb": 1131.043, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 9.773, "pushed": 9.773, "read": 9.773}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.779, "count": 13, "max_ms": 5.316, "p95_ms": 5.316}, "gstreamer_push": {"avg_ms": 1.511, "count": 13, "max_ms": 3.736, "p95_ms": 3.736}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.423, "count": 13, "max_ms": 67.692, "p95_ms": 67.692}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.099, "count": 13, "max_ms": 93.28, "p95_ms": 93.28}, "yolo11_640_fire_11class": {"avg_ms": 60.988, "count": 13, "max_ms": 79.783, "p95_ms": 79.783}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.074, "count": 13, "max_ms": 86.652, "p95_ms": 86.652}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.686, "count": 13, "max_ms": 106.773, "p95_ms": 106.773}, "read_interval": {"avg_ms": 2351.885, "count": 13, "max_ms": 29365.102, "p95_ms": 29365.102}, "triton_wait": {"avg_ms": 74.111, "count": 13, "max_ms": 93.291, "p95_ms": 93.291}, "visualize": {"avg_ms": 3.718, "count": 13, "max_ms": 4.501, "p95_ms": 4.501}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 1.33, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 9, "tracking_summary": 4582}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9592.027, "limit_mb": 8796093022207.996}, "detail": {"pid": 47090, "restart_count": 70, "start_count": 71}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 70, "h264_decode_error_count": 70, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:12:24+09:00", "last_error_summary": "[h264 @ 0x55c6cd97c7c0] error while decoding MB 52 41, bytestream -27 [in#0/rtsp @ 0x55c6cd840e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c6cd840e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T07:12:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T07:12:27+09:00", "pid": 47090, "process": {"exists": true, "fd_count": 5, "hwm_mb": 204.812, "name": "ffmpeg", "pid": 47090, "rss_mb": 204.812, "state": "S (sleeping)", "threads": 106}, "restart_count": 70, "restart_count_per_hour": 3, "rtsp_timeout_count": 126, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 71}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 110.789, "sample_count": 2322, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 549929, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:12:24+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.933, "last_frame_pushed_at": "2026-07-02T07:12:24+09:00", "last_frame_read_age_sec": 5.04, "last_frame_read_at": "2026-07-02T07:12:24+09:00", "mjpeg": {"age_sec": 4.918, "exists": true, "mtime": "2026-07-02T07:12:24.296023+09:00", "mtime_epoch": 1782943944.2960234, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 255104}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T07:12:29+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1384.668, "name": "celery", "pid": 267, "rss_mb": 1131.043, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.08, "pushed": 2.08, "read": 2.08}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.779, "count": 13, "max_ms": 5.316, "p95_ms": 5.316}, "gstreamer_push": {"avg_ms": 1.511, "count": 13, "max_ms": 3.736, "p95_ms": 3.736}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.423, "count": 13, "max_ms": 67.692, "p95_ms": 67.692}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.099, "count": 13, "max_ms": 93.28, "p95_ms": 93.28}, "yolo11_640_fire_11class": {"avg_ms": 60.988, "count": 13, "max_ms": 79.783, "p95_ms": 79.783}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.074, "count": 13, "max_ms": 86.652, "p95_ms": 86.652}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.686, "count": 13, "max_ms": 106.773, "p95_ms": 106.773}, "read_interval": {"avg_ms": 2351.885, "count": 13, "max_ms": 29365.102, "p95_ms": 29365.102}, "triton_wait": {"avg_ms": 74.111, "count": 13, "max_ms": 93.291, "p95_ms": 93.291}, "visualize": {"avg_ms": 3.718, "count": 13, "max_ms": 4.501, "p95_ms": 4.501}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 6.25, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3191}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9462.156, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 9, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x563379daa0c0] left block unavailable for requested intra mode [h264 @ 0x563379daa0c0] error while decoding MB 0 2, bytestream 4745 [h264 @ 0x56337942c740] left block unavailable for requested intra mode [h264 @ 0x56337942c740] error while decoding MB 0 22, bytestream 28760 [rtsp @ 0x5633793e3f00] RTP: PT=60: bad cseq 3352 expected=33e1 [h264 @ 0x56337a1b4880] cabac decode of qscale diff failed at 66 33 [h264 @ 0x56337a1b4880] error while decoding MB 66 33, bytestream 525 [h264 @ 0x5633", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 267, "h264_decode_error_count": 466, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:15:38+09:00", "last_error_summary": "[h264 @ 0x563379daa0c0] left block unavailable for requested intra mode [h264 @ 0x563379daa0c0] error while decoding MB 0 2, bytestream 4745 [h264 @ 0x56337942c740] left block unavailable for requested intra mode [h264 @ 0x56337942c740] error while decoding MB 0 22, bytestream 28760 [rtsp @ 0x5633793e3f00] RTP: PT=60: bad cseq 3352 expected=33e1 [h264 @ 0x56337a1b4880] cabac decode of qscale diff failed at 66 33 [h264 @ 0x56337a1b4880] error while decoding MB 66 33, bytestream 525 [h264 @ 0x5633", "last_exit_at": "2026-07-02T07:10:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 294.005, "last_started_at": "2026-07-02T07:10:44+09:00", "pid": 46983, "process": {"exists": false, "pid": 46983}, "restart_count": 299, "restart_count_per_hour": 19, "rtsp_timeout_count": 108, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 300}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 150.866, "sample_count": 2347, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 382927, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:15:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.072, "last_frame_pushed_at": "2026-07-02T07:15:38+09:00", "last_frame_read_age_sec": 0.06, "last_frame_read_at": "2026-07-02T07:15:38+09:00", "mjpeg": {"age_sec": 0.054, "exists": true, "mtime": "2026-07-02T07:15:38.675716+09:00", "mtime_epoch": 1782944138.6757157, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184668}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:15:38+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5540.523, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 97, "pushed_frames": 97, "read_frames": 97, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.238, "pushed": 3.238, "read": 3.238}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.336, "count": 97, "max_ms": 22.33, "p95_ms": 15.683}, "gstreamer_push": {"avg_ms": 1.248, "count": 97, "max_ms": 1.863, "p95_ms": 1.375}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.754, "count": 97, "max_ms": 77.457, "p95_ms": 60.41}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.571, "count": 97, "max_ms": 99.787, "p95_ms": 94.415}, "yolo11_640_fire_11class": {"avg_ms": 57.484, "count": 97, "max_ms": 76.903, "p95_ms": 74.076}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.482, "count": 97, "max_ms": 91.761, "p95_ms": 87.323}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.381, "count": 97, "max_ms": 125.684, "p95_ms": 116.42}, "read_interval": {"avg_ms": 309.124, "count": 97, "max_ms": 19584.871, "p95_ms": 133.638}, "triton_wait": {"avg_ms": 70.584, "count": 97, "max_ms": 99.8, "p95_ms": 94.427}, "visualize": {"avg_ms": 3.567, "count": 97, "max_ms": 4.707, "p95_ms": 4.094}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.961, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3191}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9461.52, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T07:15:38+09:00", "last_error_summary": "[h264 @ 0x563379daa0c0] left block unavailable for requested intra mode [h264 @ 0x563379daa0c0] error while decoding MB 0 2, bytestream 4745 [h264 @ 0x56337942c740] left block unavailable for requested intra mode [h264 @ 0x56337942c740] error while decoding MB 0 22, bytestream 28760 [rtsp @ 0x5633793e3f00] RTP: PT=60: bad cseq 3352 expected=33e1 [h264 @ 0x56337a1b4880] cabac decode of qscale diff failed at 66 33 [h264 @ 0x56337a1b4880] error while decoding MB 66 33, bytestream 525 [h264 @ 0x5633", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 267, "h264_decode_error_count": 466, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:15:38+09:00", "last_error_summary": "[h264 @ 0x563379daa0c0] left block unavailable for requested intra mode [h264 @ 0x563379daa0c0] error while decoding MB 0 2, bytestream 4745 [h264 @ 0x56337942c740] left block unavailable for requested intra mode [h264 @ 0x56337942c740] error while decoding MB 0 22, bytestream 28760 [rtsp @ 0x5633793e3f00] RTP: PT=60: bad cseq 3352 expected=33e1 [h264 @ 0x56337a1b4880] cabac decode of qscale diff failed at 66 33 [h264 @ 0x56337a1b4880] error while decoding MB 66 33, bytestream 525 [h264 @ 0x5633", "last_exit_at": "2026-07-02T07:15:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 294.053, "last_started_at": "2026-07-02T07:10:44+09:00", "pid": 46983, "process": {"exists": false, "pid": 46983}, "restart_count": 299, "restart_count_per_hour": 19, "rtsp_timeout_count": 108, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 300}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 150.866, "sample_count": 2348, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 382928, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:15:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-02T07:15:38+09:00", "last_frame_read_age_sec": 0.109, "last_frame_read_at": "2026-07-02T07:15:38+09:00", "mjpeg": {"age_sec": 0.103, "exists": true, "mtime": "2026-07-02T07:15:38.675716+09:00", "mtime_epoch": 1782944138.6757157, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184668}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:15:38+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5540.523, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 97, "pushed_frames": 97, "read_frames": 97, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.234, "pushed": 3.234, "read": 3.234}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.312, "count": 97, "max_ms": 22.33, "p95_ms": 15.612}, "gstreamer_push": {"avg_ms": 1.249, "count": 97, "max_ms": 1.863, "p95_ms": 1.375}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.874, "count": 97, "max_ms": 77.457, "p95_ms": 60.41}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.716, "count": 97, "max_ms": 99.787, "p95_ms": 94.415}, "yolo11_640_fire_11class": {"avg_ms": 57.663, "count": 97, "max_ms": 76.903, "p95_ms": 74.076}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.654, "count": 97, "max_ms": 91.761, "p95_ms": 87.323}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.511, "count": 97, "max_ms": 125.684, "p95_ms": 116.42}, "read_interval": {"avg_ms": 309.124, "count": 97, "max_ms": 19584.871, "p95_ms": 133.638}, "triton_wait": {"avg_ms": 70.729, "count": 97, "max_ms": 99.8, "p95_ms": 94.427}, "visualize": {"avg_ms": 3.567, "count": 97, "max_ms": 4.707, "p95_ms": 4.094}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3191}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9461.449, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 267, "h264_decode_error_count": 466, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:15:38+09:00", "last_error_summary": "[h264 @ 0x563379daa0c0] left block unavailable for requested intra mode [h264 @ 0x563379daa0c0] error while decoding MB 0 2, bytestream 4745 [h264 @ 0x56337942c740] left block unavailable for requested intra mode [h264 @ 0x56337942c740] error while decoding MB 0 22, bytestream 28760 [rtsp @ 0x5633793e3f00] RTP: PT=60: bad cseq 3352 expected=33e1 [h264 @ 0x56337a1b4880] cabac decode of qscale diff failed at 66 33 [h264 @ 0x56337a1b4880] error while decoding MB 66 33, bytestream 525 [h264 @ 0x5633", "last_exit_at": "2026-07-02T07:15:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 294.063, "last_started_at": "2026-07-02T07:10:44+09:00", "pid": 46983, "process": {"exists": false, "pid": 46983}, "restart_count": 299, "restart_count_per_hour": 19, "rtsp_timeout_count": 108, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 300}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 150.866, "sample_count": 2348, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 382928, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:15:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.019, "last_frame_pushed_at": "2026-07-02T07:15:38+09:00", "last_frame_read_age_sec": 0.119, "last_frame_read_at": "2026-07-02T07:15:38+09:00", "mjpeg": {"age_sec": 0.113, "exists": true, "mtime": "2026-07-02T07:15:38.675716+09:00", "mtime_epoch": 1782944138.6757157, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184668}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:15:38+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5540.523, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 97, "pushed_frames": 97, "read_frames": 96, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.242, "pushed": 3.242, "read": 3.209}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.312, "count": 97, "max_ms": 22.33, "p95_ms": 15.612}, "gstreamer_push": {"avg_ms": 1.249, "count": 97, "max_ms": 1.863, "p95_ms": 1.375}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.874, "count": 97, "max_ms": 77.457, "p95_ms": 60.41}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.716, "count": 97, "max_ms": 99.787, "p95_ms": 94.415}, "yolo11_640_fire_11class": {"avg_ms": 57.663, "count": 97, "max_ms": 76.903, "p95_ms": 74.076}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.654, "count": 97, "max_ms": 91.761, "p95_ms": 87.323}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.511, "count": 97, "max_ms": 125.684, "p95_ms": 116.42}, "read_interval": {"avg_ms": 311.311, "count": 96, "max_ms": 19584.871, "p95_ms": 133.638}, "triton_wait": {"avg_ms": 70.729, "count": 97, "max_ms": 99.8, "p95_ms": 94.427}, "visualize": {"avg_ms": 3.567, "count": 97, "max_ms": 4.707, "p95_ms": 4.094}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.916, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3191}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9569.0, "limit_mb": 8796093022207.996}, "detail": {"pid": 47242, "restart_count": 300, "start_count": 301}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 267, "h264_decode_error_count": 466, "h264_decode_error_count_per_hour": 32, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:15:38+09:00", "last_error_summary": "[h264 @ 0x563379daa0c0] left block unavailable for requested intra mode [h264 @ 0x563379daa0c0] error while decoding MB 0 2, bytestream 4745 [h264 @ 0x56337942c740] left block unavailable for requested intra mode [h264 @ 0x56337942c740] error while decoding MB 0 22, bytestream 28760 [rtsp @ 0x5633793e3f00] RTP: PT=60: bad cseq 3352 expected=33e1 [h264 @ 0x56337a1b4880] cabac decode of qscale diff failed at 66 33 [h264 @ 0x56337a1b4880] error while decoding MB 66 33, bytestream 525 [h264 @ 0x5633", "last_exit_at": "2026-07-02T07:15:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T07:15:41+09:00", "pid": 47242, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.836, "name": "ffmpeg", "pid": 47242, "rss_mb": 138.836, "state": "S (sleeping)", "threads": 106}, "restart_count": 300, "restart_count_per_hour": 20, "rtsp_timeout_count": 108, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 301}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 150.924, "sample_count": 2321, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 382928, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:15:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.557, "last_frame_pushed_at": "2026-07-02T07:15:38+09:00", "last_frame_read_age_sec": 4.657, "last_frame_read_at": "2026-07-02T07:15:38+09:00", "mjpeg": {"age_sec": 4.539, "exists": true, "mtime": "2026-07-02T07:15:38.787718+09:00", "mtime_epoch": 1782944138.7877178, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185329}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:15:43+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5540.523, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 55, "pushed_frames": 55, "read_frames": 54, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.837, "pushed": 1.837, "read": 1.804}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.192, "count": 55, "max_ms": 15.683, "p95_ms": 15.082}, "gstreamer_push": {"avg_ms": 1.239, "count": 55, "max_ms": 1.443, "p95_ms": 1.373}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.026, "count": 55, "max_ms": 77.457, "p95_ms": 60.607}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.216, "count": 55, "max_ms": 94.587, "p95_ms": 93.327}, "yolo11_640_fire_11class": {"avg_ms": 58.968, "count": 55, "max_ms": 76.903, "p95_ms": 74.311}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.743, "count": 55, "max_ms": 87.323, "p95_ms": 86.03}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.86, "count": 55, "max_ms": 116.42, "p95_ms": 116.001}, "read_interval": {"avg_ms": 470.329, "count": 54, "max_ms": 19584.871, "p95_ms": 131.297}, "triton_wait": {"avg_ms": 72.23, "count": 55, "max_ms": 94.6, "p95_ms": 93.339}, "visualize": {"avg_ms": 3.532, "count": 55, "max_ms": 4.274, "p95_ms": 3.832}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.94, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3208}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9464.742, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x564dc0063640] error while decoding MB 49 11, bytestream -11", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 268, "h264_decode_error_count": 467, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:20:01+09:00", "last_error_summary": "[h264 @ 0x564dc0063640] error while decoding MB 49 11, bytestream -11", "last_exit_at": "2026-07-02T07:15:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 260.144, "last_started_at": "2026-07-02T07:15:41+09:00", "pid": 47242, "process": {"exists": false, "pid": 47242}, "restart_count": 300, "restart_count_per_hour": 18, "rtsp_timeout_count": 108, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 301}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 149.244, "sample_count": 2214, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 384978, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:20:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.064, "last_frame_pushed_at": "2026-07-02T07:20:01+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-02T07:20:01+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-02T07:20:01.424655+09:00", "mtime_epoch": 1782944401.4246547, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200026}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:20:01+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5541.453, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 201, "pushed_frames": 201, "read_frames": 201, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.727, "pushed": 6.727, "read": 6.727}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 16.391, "count": 201, "max_ms": 28.173, "p95_ms": 17.679}, "gstreamer_push": {"avg_ms": 1.132, "count": 201, "max_ms": 1.345, "p95_ms": 1.208}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.203, "count": 201, "max_ms": 71.358, "p95_ms": 66.876}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.129, "count": 201, "max_ms": 94.414, "p95_ms": 91.16}, "yolo11_640_fire_11class": {"avg_ms": 58.849, "count": 201, "max_ms": 73.658, "p95_ms": 70.347}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.79, "count": 201, "max_ms": 93.506, "p95_ms": 87.424}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.84, "count": 201, "max_ms": 177.802, "p95_ms": 152.258}, "read_interval": {"avg_ms": 148.993, "count": 201, "max_ms": 189.552, "p95_ms": 165.854}, "triton_wait": {"avg_ms": 74.141, "count": 201, "max_ms": 94.427, "p95_ms": 91.173}, "visualize": {"avg_ms": 4.95, "count": 201, "max_ms": 6.015, "p95_ms": 5.718}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.878, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3208}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9463.648, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T07:20:01+09:00", "last_error_summary": "[h264 @ 0x564dc0063640] error while decoding MB 49 11, bytestream -11", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 268, "h264_decode_error_count": 467, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:20:01+09:00", "last_error_summary": "[h264 @ 0x564dc0063640] error while decoding MB 49 11, bytestream -11", "last_exit_at": "2026-07-02T07:20:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 260.254, "last_started_at": "2026-07-02T07:15:41+09:00", "pid": 47242, "process": {"exists": false, "pid": 47242}, "restart_count": 300, "restart_count_per_hour": 18, "rtsp_timeout_count": 108, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 301}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 149.293, "sample_count": 2215, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 384979, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:20:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T07:20:01+09:00", "last_frame_read_age_sec": 0.163, "last_frame_read_at": "2026-07-02T07:20:01+09:00", "mjpeg": {"age_sec": 0.154, "exists": true, "mtime": "2026-07-02T07:20:01.424655+09:00", "mtime_epoch": 1782944401.4246547, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200026}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:20:01+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5541.453, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 202, "pushed_frames": 202, "read_frames": 201, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.736, "pushed": 6.736, "read": 6.703}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 16.392, "count": 201, "max_ms": 28.173, "p95_ms": 17.679}, "gstreamer_push": {"avg_ms": 1.132, "count": 202, "max_ms": 1.345, "p95_ms": 1.208}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.343, "count": 201, "max_ms": 71.358, "p95_ms": 67.037}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.281, "count": 201, "max_ms": 94.414, "p95_ms": 91.979}, "yolo11_640_fire_11class": {"avg_ms": 58.954, "count": 201, "max_ms": 73.658, "p95_ms": 70.422}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.962, "count": 201, "max_ms": 93.506, "p95_ms": 87.591}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.94, "count": 202, "max_ms": 177.802, "p95_ms": 152.538}, "read_interval": {"avg_ms": 148.993, "count": 201, "max_ms": 189.552, "p95_ms": 165.854}, "triton_wait": {"avg_ms": 74.294, "count": 201, "max_ms": 94.427, "p95_ms": 91.992}, "visualize": {"avg_ms": 4.95, "count": 201, "max_ms": 6.015, "p95_ms": 5.718}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.988, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3208}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9463.641, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 268, "h264_decode_error_count": 467, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:20:01+09:00", "last_error_summary": "[h264 @ 0x564dc0063640] error while decoding MB 49 11, bytestream -11", "last_exit_at": "2026-07-02T07:20:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 260.26, "last_started_at": "2026-07-02T07:15:41+09:00", "pid": 47242, "process": {"exists": false, "pid": 47242}, "restart_count": 300, "restart_count_per_hour": 18, "rtsp_timeout_count": 108, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 301}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 149.293, "sample_count": 2214, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 384979, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:20:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-02T07:20:01+09:00", "last_frame_read_age_sec": 0.169, "last_frame_read_at": "2026-07-02T07:20:01+09:00", "mjpeg": {"age_sec": 0.161, "exists": true, "mtime": "2026-07-02T07:20:01.424655+09:00", "mtime_epoch": 1782944401.4246547, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200026}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:20:01+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5541.453, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 202, "pushed_frames": 202, "read_frames": 201, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.735, "pushed": 6.735, "read": 6.701}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 16.392, "count": 201, "max_ms": 28.173, "p95_ms": 17.679}, "gstreamer_push": {"avg_ms": 1.132, "count": 202, "max_ms": 1.345, "p95_ms": 1.208}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.343, "count": 201, "max_ms": 71.358, "p95_ms": 67.037}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.281, "count": 201, "max_ms": 94.414, "p95_ms": 91.979}, "yolo11_640_fire_11class": {"avg_ms": 58.954, "count": 201, "max_ms": 73.658, "p95_ms": 70.422}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.962, "count": 201, "max_ms": 93.506, "p95_ms": 87.591}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.94, "count": 202, "max_ms": 177.802, "p95_ms": 152.538}, "read_interval": {"avg_ms": 148.993, "count": 201, "max_ms": 189.552, "p95_ms": 165.854}, "triton_wait": {"avg_ms": 74.294, "count": 201, "max_ms": 94.427, "p95_ms": 91.992}, "visualize": {"avg_ms": 4.95, "count": 201, "max_ms": 6.015, "p95_ms": 5.718}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3208}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9574.848, "limit_mb": 8796093022207.996}, "detail": {"pid": 47349, "restart_count": 301, "start_count": 302}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 268, "h264_decode_error_count": 467, "h264_decode_error_count_per_hour": 31, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:20:01+09:00", "last_error_summary": "[h264 @ 0x564dc0063640] error while decoding MB 49 11, bytestream -11", "last_exit_at": "2026-07-02T07:20:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T07:20:03+09:00", "pid": 47349, "process": {"exists": true, "fd_count": 5, "hwm_mb": 134.23, "name": "ffmpeg", "pid": 47349, "rss_mb": 134.23, "state": "R (running)", "threads": 106}, "restart_count": 301, "restart_count_per_hour": 19, "rtsp_timeout_count": 108, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 302}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 149.459, "sample_count": 2175, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 384979, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:20:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.211, "last_frame_pushed_at": "2026-07-02T07:20:01+09:00", "last_frame_read_age_sec": 4.369, "last_frame_read_at": "2026-07-02T07:20:01+09:00", "mjpeg": {"age_sec": 4.196, "exists": true, "mtime": "2026-07-02T07:20:01.588658+09:00", "mtime_epoch": 1782944401.5886579, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200193}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:20:05+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5541.453, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 173, "pushed_frames": 173, "read_frames": 172, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.782, "pushed": 5.782, "read": 5.748}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 16.457, "count": 173, "max_ms": 28.173, "p95_ms": 17.912}, "gstreamer_push": {"avg_ms": 1.133, "count": 173, "max_ms": 1.345, "p95_ms": 1.23}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.363, "count": 173, "max_ms": 71.358, "p95_ms": 67.121}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.393, "count": 173, "max_ms": 94.414, "p95_ms": 92.341}, "yolo11_640_fire_11class": {"avg_ms": 58.972, "count": 173, "max_ms": 73.658, "p95_ms": 70.422}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.067, "count": 173, "max_ms": 93.506, "p95_ms": 87.794}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.37, "count": 173, "max_ms": 177.802, "p95_ms": 152.904}, "read_interval": {"avg_ms": 149.385, "count": 172, "max_ms": 189.552, "p95_ms": 165.854}, "triton_wait": {"avg_ms": 74.405, "count": 173, "max_ms": 94.427, "p95_ms": 92.354}, "visualize": {"avg_ms": 4.959, "count": 173, "max_ms": 6.015, "p95_ms": 5.745}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.921, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 9, "tracking_summary": 4616}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9411.891, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x559b721d04c0] error while decoding MB 12 48, bytestream -15 [in#0/rtsp @ 0x559b720cbe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559b720cbe00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 71, "h264_decode_error_count": 71, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:21:03+09:00", "last_error_summary": "[h264 @ 0x559b721d04c0] error while decoding MB 12 48, bytestream -15 [in#0/rtsp @ 0x559b720cbe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559b720cbe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T07:12:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 516.198, "last_started_at": "2026-07-02T07:12:27+09:00", "pid": 47090, "process": {"exists": false, "pid": 47090}, "restart_count": 70, "restart_count_per_hour": 3, "rtsp_timeout_count": 128, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 71}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 110.48, "sample_count": 2485, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 554030, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:21:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-02T07:21:03+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-02T07:21:03+09:00", "mjpeg": {"age_sec": 0.057, "exists": true, "mtime": "2026-07-02T07:21:03.353813+09:00", "mtime_epoch": 1782944463.3538127, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 258426}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T07:21:03+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1384.668, "name": "celery", "pid": 267, "rss_mb": 1131.074, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 91, "pushed_frames": 91, "read_frames": 91, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.043, "pushed": 3.043, "read": 3.043}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.206, "count": 91, "max_ms": 6.353, "p95_ms": 5.889}, "gstreamer_push": {"avg_ms": 1.598, "count": 91, "max_ms": 6.057, "p95_ms": 3.737}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.252, "count": 91, "max_ms": 77.772, "p95_ms": 75.014}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 92.227, "count": 91, "max_ms": 102.619, "p95_ms": 97.96}, "yolo11_640_fire_11class": {"avg_ms": 70.143, "count": 91, "max_ms": 78.318, "p95_ms": 76.363}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.146, "count": 91, "max_ms": 94.383, "p95_ms": 89.625}}, "truncated_count": 0}, "processing_total": {"avg_ms": 106.219, "count": 91, "max_ms": 116.14, "p95_ms": 111.654}, "read_interval": {"avg_ms": 329.142, "count": 91, "max_ms": 10164.593, "p95_ms": 131.579}, "triton_wait": {"avg_ms": 92.241, "count": 91, "max_ms": 102.634, "p95_ms": 97.974}, "visualize": {"avg_ms": 3.679, "count": 91, "max_ms": 4.466, "p95_ms": 3.93}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.901, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 9, "tracking_summary": 4616}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9411.371, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T07:21:03+09:00", "last_error_summary": "[h264 @ 0x559b721d04c0] error while decoding MB 12 48, bytestream -15 [in#0/rtsp @ 0x559b720cbe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559b720cbe00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 71, "h264_decode_error_count": 71, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:21:03+09:00", "last_error_summary": "[h264 @ 0x559b721d04c0] error while decoding MB 12 48, bytestream -15 [in#0/rtsp @ 0x559b720cbe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559b720cbe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T07:21:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 516.236, "last_started_at": "2026-07-02T07:12:27+09:00", "pid": 47090, "process": {"exists": false, "pid": 47090}, "restart_count": 70, "restart_count_per_hour": 3, "rtsp_timeout_count": 128, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 71}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 110.48, "sample_count": 2486, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 554031, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:21:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T07:21:03+09:00", "last_frame_read_age_sec": 0.103, "last_frame_read_at": "2026-07-02T07:21:03+09:00", "mjpeg": {"age_sec": 0.095, "exists": true, "mtime": "2026-07-02T07:21:03.353813+09:00", "mtime_epoch": 1782944463.3538127, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 258426}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T07:21:03+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1384.668, "name": "celery", "pid": 267, "rss_mb": 1131.074, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 92, "pushed_frames": 92, "read_frames": 91, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.073, "pushed": 3.073, "read": 3.04}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.202, "count": 92, "max_ms": 6.353, "p95_ms": 5.889}, "gstreamer_push": {"avg_ms": 1.596, "count": 92, "max_ms": 6.057, "p95_ms": 3.737}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.193, "count": 92, "max_ms": 77.772, "p95_ms": 75.014}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 92.049, "count": 92, "max_ms": 102.619, "p95_ms": 97.96}, "yolo11_640_fire_11class": {"avg_ms": 70.049, "count": 92, "max_ms": 78.318, "p95_ms": 76.363}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.131, "count": 92, "max_ms": 94.383, "p95_ms": 89.625}}, "truncated_count": 0}, "processing_total": {"avg_ms": 106.089, "count": 92, "max_ms": 116.14, "p95_ms": 111.654}, "read_interval": {"avg_ms": 329.142, "count": 91, "max_ms": 10164.593, "p95_ms": 131.579}, "triton_wait": {"avg_ms": 92.116, "count": 92, "max_ms": 102.634, "p95_ms": 97.974}, "visualize": {"avg_ms": 3.678, "count": 92, "max_ms": 4.466, "p95_ms": 3.93}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.939, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 9, "tracking_summary": 4616}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9411.363, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 71, "h264_decode_error_count": 71, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:21:03+09:00", "last_error_summary": "[h264 @ 0x559b721d04c0] error while decoding MB 12 48, bytestream -15 [in#0/rtsp @ 0x559b720cbe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559b720cbe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T07:21:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 516.244, "last_started_at": "2026-07-02T07:12:27+09:00", "pid": 47090, "process": {"exists": false, "pid": 47090}, "restart_count": 70, "restart_count_per_hour": 3, "rtsp_timeout_count": 128, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 71}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 110.48, "sample_count": 2486, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 554031, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:21:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-02T07:21:03+09:00", "last_frame_read_age_sec": 0.111, "last_frame_read_at": "2026-07-02T07:21:03+09:00", "mjpeg": {"age_sec": 0.103, "exists": true, "mtime": "2026-07-02T07:21:03.353813+09:00", "mtime_epoch": 1782944463.3538127, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 258426}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T07:21:03+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1384.668, "name": "celery", "pid": 267, "rss_mb": 1131.074, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 92, "pushed_frames": 92, "read_frames": 91, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.072, "pushed": 3.072, "read": 3.039}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.202, "count": 92, "max_ms": 6.353, "p95_ms": 5.889}, "gstreamer_push": {"avg_ms": 1.596, "count": 92, "max_ms": 6.057, "p95_ms": 3.737}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.193, "count": 92, "max_ms": 77.772, "p95_ms": 75.014}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 92.049, "count": 92, "max_ms": 102.619, "p95_ms": 97.96}, "yolo11_640_fire_11class": {"avg_ms": 70.049, "count": 92, "max_ms": 78.318, "p95_ms": 76.363}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.131, "count": 92, "max_ms": 94.383, "p95_ms": 89.625}}, "truncated_count": 0}, "processing_total": {"avg_ms": 106.089, "count": 92, "max_ms": 116.14, "p95_ms": 111.654}, "read_interval": {"avg_ms": 329.142, "count": 91, "max_ms": 10164.593, "p95_ms": 131.579}, "triton_wait": {"avg_ms": 92.116, "count": 92, "max_ms": 102.634, "p95_ms": 97.974}, "visualize": {"avg_ms": 3.678, "count": 92, "max_ms": 4.466, "p95_ms": 3.93}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.946, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 9, "tracking_summary": 4616}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9432.328, "limit_mb": 8796093022207.996}, "detail": {"pid": 47456, "restart_count": 71, "start_count": 72}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 71, "h264_decode_error_count": 71, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:21:03+09:00", "last_error_summary": "[h264 @ 0x559b721d04c0] error while decoding MB 12 48, bytestream -15 [in#0/rtsp @ 0x559b720cbe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559b720cbe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T07:21:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T07:21:05+09:00", "pid": 47456, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.34, "name": "ffmpeg", "pid": 47456, "rss_mb": 58.34, "state": "S (sleeping)", "threads": 1}, "restart_count": 71, "restart_count_per_hour": 4, "rtsp_timeout_count": 128, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 72}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 110.463, "sample_count": 2450, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 554031, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:21:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.397, "last_frame_pushed_at": "2026-07-02T07:21:03+09:00", "last_frame_read_age_sec": 4.493, "last_frame_read_at": "2026-07-02T07:21:03+09:00", "mjpeg": {"age_sec": 4.381, "exists": true, "mtime": "2026-07-02T07:21:03.457815+09:00", "mtime_epoch": 1782944463.4578147, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 256642}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T07:21:07+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1384.668, "name": "celery", "pid": 267, "rss_mb": 1131.074, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 57, "pushed_frames": 57, "read_frames": 56, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.902, "pushed": 1.902, "read": 1.869}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.287, "count": 57, "max_ms": 6.353, "p95_ms": 5.922}, "gstreamer_push": {"avg_ms": 1.64, "count": 57, "max_ms": 3.917, "p95_ms": 3.737}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.685, "count": 57, "max_ms": 77.772, "p95_ms": 76.742}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 90.321, "count": 57, "max_ms": 98.898, "p95_ms": 96.771}, "yolo11_640_fire_11class": {"avg_ms": 70.324, "count": 57, "max_ms": 78.318, "p95_ms": 76.742}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 81.027, "count": 57, "max_ms": 92.341, "p95_ms": 89.536}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.502, "count": 57, "max_ms": 112.559, "p95_ms": 110.787}, "read_interval": {"avg_ms": 456.772, "count": 56, "max_ms": 10164.593, "p95_ms": 133.814}, "triton_wait": {"avg_ms": 90.422, "count": 57, "max_ms": 98.912, "p95_ms": 96.784}, "visualize": {"avg_ms": 3.7, "count": 57, "max_ms": 4.466, "p95_ms": 3.971}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.966, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3226}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9471.0, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 5, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x559a2ad51940] error while decoding MB 47 28, bytestream -8 [h264 @ 0x559a2ada1f80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559a2ada1f80] error while decoding MB 0 26, bytestream 4372 [h264 @ 0x559a2b7b4b40] top block unavailable for requested intra mode -1 [h264 @ 0x559a2b7b4b40] error while decoding MB 55 0, bytestream 2098 [h264 @ 0x559a2ac990c0] left block unavailable for requested intra mode [h264 @ 0x559a2ac990c0] error while decoding MB 0 20, bytestream 85", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 269, "h264_decode_error_count": 472, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:25:03+09:00", "last_error_summary": "[h264 @ 0x559a2ad51940] error while decoding MB 47 28, bytestream -8 [h264 @ 0x559a2ada1f80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559a2ada1f80] error while decoding MB 0 26, bytestream 4372 [h264 @ 0x559a2b7b4b40] top block unavailable for requested intra mode -1 [h264 @ 0x559a2b7b4b40] error while decoding MB 55 0, bytestream 2098 [h264 @ 0x559a2ac990c0] left block unavailable for requested intra mode [h264 @ 0x559a2ac990c0] error while decoding MB 0 20, bytestream 85", "last_exit_at": "2026-07-02T07:20:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 299.27, "last_started_at": "2026-07-02T07:20:03+09:00", "pid": 47349, "process": {"exists": false, "pid": 47349}, "restart_count": 301, "restart_count_per_hour": 17, "rtsp_timeout_count": 110, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 302}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 125.614, "sample_count": 2221, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 387200, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:25:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.079, "last_frame_pushed_at": "2026-07-02T07:25:02+09:00", "last_frame_read_age_sec": 0.068, "last_frame_read_at": "2026-07-02T07:25:02+09:00", "mjpeg": {"age_sec": 0.059, "exists": true, "mtime": "2026-07-02T07:25:02.994278+09:00", "mtime_epoch": 1782944702.9942782, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189411}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:25:03+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5542.566, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 93, "pushed_frames": 93, "read_frames": 94, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.1, "pushed": 3.1, "read": 3.134}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.208, "count": 93, "max_ms": 15.666, "p95_ms": 14.728}, "gstreamer_push": {"avg_ms": 1.295, "count": 93, "max_ms": 1.404, "p95_ms": 1.383}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.026, "count": 93, "max_ms": 71.432, "p95_ms": 65.742}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.763, "count": 93, "max_ms": 94.583, "p95_ms": 90.673}, "yolo11_640_fire_11class": {"avg_ms": 60.547, "count": 93, "max_ms": 74.014, "p95_ms": 68.029}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.646, "count": 93, "max_ms": 86.011, "p95_ms": 81.985}}, "truncated_count": 0}, "processing_total": {"avg_ms": 101.546, "count": 93, "max_ms": 117.546, "p95_ms": 113.617}, "read_interval": {"avg_ms": 319.472, "count": 94, "max_ms": 19178.022, "p95_ms": 129.478}, "triton_wait": {"avg_ms": 78.776, "count": 93, "max_ms": 94.6, "p95_ms": 90.685}, "visualize": {"avg_ms": 3.591, "count": 93, "max_ms": 4.325, "p95_ms": 3.827}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3226}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9470.766, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T07:25:03+09:00", "last_error_summary": "[h264 @ 0x559a2ad51940] error while decoding MB 47 28, bytestream -8 [h264 @ 0x559a2ada1f80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559a2ada1f80] error while decoding MB 0 26, bytestream 4372 [h264 @ 0x559a2b7b4b40] top block unavailable for requested intra mode -1 [h264 @ 0x559a2b7b4b40] error while decoding MB 55 0, bytestream 2098 [h264 @ 0x559a2ac990c0] left block unavailable for requested intra mode [h264 @ 0x559a2ac990c0] error while decoding MB 0 20, bytestream 85", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 269, "h264_decode_error_count": 472, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:25:03+09:00", "last_error_summary": "[h264 @ 0x559a2ad51940] error while decoding MB 47 28, bytestream -8 [h264 @ 0x559a2ada1f80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559a2ada1f80] error while decoding MB 0 26, bytestream 4372 [h264 @ 0x559a2b7b4b40] top block unavailable for requested intra mode -1 [h264 @ 0x559a2b7b4b40] error while decoding MB 55 0, bytestream 2098 [h264 @ 0x559a2ac990c0] left block unavailable for requested intra mode [h264 @ 0x559a2ac990c0] error while decoding MB 0 20, bytestream 85", "last_exit_at": "2026-07-02T07:25:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 299.293, "last_started_at": "2026-07-02T07:20:03+09:00", "pid": 47349, "process": {"exists": false, "pid": 47349}, "restart_count": 301, "restart_count_per_hour": 17, "rtsp_timeout_count": 110, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 302}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 125.614, "sample_count": 2222, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 387201, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:25:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T07:25:03+09:00", "last_frame_read_age_sec": 0.091, "last_frame_read_at": "2026-07-02T07:25:02+09:00", "mjpeg": {"age_sec": 0.083, "exists": true, "mtime": "2026-07-02T07:25:02.994278+09:00", "mtime_epoch": 1782944702.9942782, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189411}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:25:03+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5542.566, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 94, "pushed_frames": 94, "read_frames": 93, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.14, "pushed": 3.14, "read": 3.107}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.231, "count": 94, "max_ms": 16.318, "p95_ms": 14.838}, "gstreamer_push": {"avg_ms": 1.296, "count": 94, "max_ms": 1.404, "p95_ms": 1.383}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.841, "count": 94, "max_ms": 71.432, "p95_ms": 65.742}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.526, "count": 94, "max_ms": 94.583, "p95_ms": 90.673}, "yolo11_640_fire_11class": {"avg_ms": 60.4, "count": 94, "max_ms": 74.014, "p95_ms": 68.029}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.429, "count": 94, "max_ms": 86.011, "p95_ms": 81.985}}, "truncated_count": 0}, "processing_total": {"avg_ms": 101.351, "count": 94, "max_ms": 117.546, "p95_ms": 113.617}, "read_interval": {"avg_ms": 321.8, "count": 93, "max_ms": 19178.022, "p95_ms": 129.478}, "triton_wait": {"avg_ms": 78.539, "count": 94, "max_ms": 94.6, "p95_ms": 90.685}, "visualize": {"avg_ms": 3.607, "count": 94, "max_ms": 5.108, "p95_ms": 3.839}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.933, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3226}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9470.867, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 269, "h264_decode_error_count": 472, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:25:03+09:00", "last_error_summary": "[h264 @ 0x559a2ad51940] error while decoding MB 47 28, bytestream -8 [h264 @ 0x559a2ada1f80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559a2ada1f80] error while decoding MB 0 26, bytestream 4372 [h264 @ 0x559a2b7b4b40] top block unavailable for requested intra mode -1 [h264 @ 0x559a2b7b4b40] error while decoding MB 55 0, bytestream 2098 [h264 @ 0x559a2ac990c0] left block unavailable for requested intra mode [h264 @ 0x559a2ac990c0] error while decoding MB 0 20, bytestream 85", "last_exit_at": "2026-07-02T07:25:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 299.301, "last_started_at": "2026-07-02T07:20:03+09:00", "pid": 47349, "process": {"exists": false, "pid": 47349}, "restart_count": 301, "restart_count_per_hour": 17, "rtsp_timeout_count": 110, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 302}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 125.614, "sample_count": 2222, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 387201, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:25:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-02T07:25:03+09:00", "last_frame_read_age_sec": 0.099, "last_frame_read_at": "2026-07-02T07:25:02+09:00", "mjpeg": {"age_sec": 0.09, "exists": true, "mtime": "2026-07-02T07:25:02.994278+09:00", "mtime_epoch": 1782944702.9942782, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189411}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:25:03+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5542.566, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 94, "pushed_frames": 94, "read_frames": 93, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.14, "pushed": 3.14, "read": 3.106}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.231, "count": 94, "max_ms": 16.318, "p95_ms": 14.838}, "gstreamer_push": {"avg_ms": 1.296, "count": 94, "max_ms": 1.404, "p95_ms": 1.383}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.841, "count": 94, "max_ms": 71.432, "p95_ms": 65.742}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.526, "count": 94, "max_ms": 94.583, "p95_ms": 90.673}, "yolo11_640_fire_11class": {"avg_ms": 60.4, "count": 94, "max_ms": 74.014, "p95_ms": 68.029}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.429, "count": 94, "max_ms": 86.011, "p95_ms": 81.985}}, "truncated_count": 0}, "processing_total": {"avg_ms": 101.351, "count": 94, "max_ms": 117.546, "p95_ms": 113.617}, "read_interval": {"avg_ms": 321.8, "count": 93, "max_ms": 19178.022, "p95_ms": 129.478}, "triton_wait": {"avg_ms": 78.539, "count": 94, "max_ms": 94.6, "p95_ms": 90.685}, "visualize": {"avg_ms": 3.607, "count": 94, "max_ms": 5.108, "p95_ms": 3.839}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.94, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3226}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9577.324, "limit_mb": 8796093022207.996}, "detail": {"pid": 47608, "restart_count": 302, "start_count": 303}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 269, "h264_decode_error_count": 472, "h264_decode_error_count_per_hour": 34, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:25:03+09:00", "last_error_summary": "[h264 @ 0x559a2ad51940] error while decoding MB 47 28, bytestream -8 [h264 @ 0x559a2ada1f80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x559a2ada1f80] error while decoding MB 0 26, bytestream 4372 [h264 @ 0x559a2b7b4b40] top block unavailable for requested intra mode -1 [h264 @ 0x559a2b7b4b40] error while decoding MB 55 0, bytestream 2098 [h264 @ 0x559a2ac990c0] left block unavailable for requested intra mode [h264 @ 0x559a2ac990c0] error while decoding MB 0 20, bytestream 85", "last_exit_at": "2026-07-02T07:25:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T07:25:05+09:00", "pid": 47608, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.234, "name": "ffmpeg", "pid": 47608, "rss_mb": 136.234, "state": "S (sleeping)", "threads": 106}, "restart_count": 302, "restart_count_per_hour": 18, "rtsp_timeout_count": 110, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 303}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 125.614, "sample_count": 2205, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 387201, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:25:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.834, "last_frame_pushed_at": "2026-07-02T07:25:03+09:00", "last_frame_read_age_sec": 4.919, "last_frame_read_at": "2026-07-02T07:25:02+09:00", "mjpeg": {"age_sec": 4.818, "exists": true, "mtime": "2026-07-02T07:25:03.086280+09:00", "mtime_epoch": 1782944703.0862799, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190287}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:25:07+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5542.566, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 53, "pushed_frames": 53, "read_frames": 52, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.768, "pushed": 1.768, "read": 1.734}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.196, "count": 53, "max_ms": 16.318, "p95_ms": 14.654}, "gstreamer_push": {"avg_ms": 1.295, "count": 53, "max_ms": 1.386, "p95_ms": 1.368}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.965, "count": 53, "max_ms": 71.432, "p95_ms": 67.508}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.193, "count": 53, "max_ms": 92.774, "p95_ms": 90.673}, "yolo11_640_fire_11class": {"avg_ms": 60.349, "count": 53, "max_ms": 70.893, "p95_ms": 68.029}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.483, "count": 53, "max_ms": 86.011, "p95_ms": 82.73}}, "truncated_count": 0}, "processing_total": {"avg_ms": 101.941, "count": 53, "max_ms": 115.707, "p95_ms": 113.617}, "read_interval": {"avg_ms": 483.984, "count": 52, "max_ms": 19178.022, "p95_ms": 129.112}, "triton_wait": {"avg_ms": 79.206, "count": 53, "max_ms": 92.786, "p95_ms": 90.685}, "visualize": {"avg_ms": 3.563, "count": 53, "max_ms": 5.108, "p95_ms": 3.786}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.983, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 9, "tracking_summary": 4633}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9397.359, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55f3f0323440] error while decoding MB 52 55, bytestream -5 [in#0/rtsp @ 0x55f3f019ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f3f019ae00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 72, "h264_decode_error_count": 72, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:25:23+09:00", "last_error_summary": "[h264 @ 0x55f3f0323440] error while decoding MB 52 55, bytestream -5 [in#0/rtsp @ 0x55f3f019ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f3f019ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T07:21:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 257.181, "last_started_at": "2026-07-02T07:21:05+09:00", "pid": 47456, "process": {"exists": false, "pid": 47456}, "restart_count": 71, "restart_count_per_hour": 4, "rtsp_timeout_count": 130, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 72}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.125, "sample_count": 2207, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 556060, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:25:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.076, "last_frame_pushed_at": "2026-07-02T07:25:22+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-02T07:25:22+09:00", "mjpeg": {"age_sec": 0.056, "exists": true, "mtime": "2026-07-02T07:25:22.962683+09:00", "mtime_epoch": 1782944722.9626834, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 264008}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T07:25:23+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1384.668, "name": "celery", "pid": 267, "rss_mb": 1127.105, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 61, "pushed_frames": 61, "read_frames": 62, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.034, "pushed": 2.034, "read": 2.068}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.206, "count": 61, "max_ms": 7.479, "p95_ms": 7.174}, "gstreamer_push": {"avg_ms": 1.42, "count": 61, "max_ms": 3.846, "p95_ms": 1.5}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.132, "count": 61, "max_ms": 58.299, "p95_ms": 50.715}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.173, "count": 61, "max_ms": 101.525, "p95_ms": 95.949}, "yolo11_640_fire_11class": {"avg_ms": 49.186, "count": 61, "max_ms": 80.691, "p95_ms": 72.519}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.903, "count": 61, "max_ms": 87.895, "p95_ms": 71.132}}, "truncated_count": 0}, "processing_total": {"avg_ms": 78.176, "count": 61, "max_ms": 149.093, "p95_ms": 113.196}, "read_interval": {"avg_ms": 483.967, "count": 62, "max_ms": 15089.743, "p95_ms": 128.977}, "triton_wait": {"avg_ms": 61.186, "count": 61, "max_ms": 101.539, "p95_ms": 95.962}, "visualize": {"avg_ms": 3.766, "count": 61, "max_ms": 5.18, "p95_ms": 5.08}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.988, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 9, "tracking_summary": 4633}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9399.316, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T07:25:23+09:00", "last_error_summary": "[h264 @ 0x55f3f0323440] error while decoding MB 52 55, bytestream -5 [in#0/rtsp @ 0x55f3f019ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f3f019ae00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 72, "h264_decode_error_count": 72, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:25:23+09:00", "last_error_summary": "[h264 @ 0x55f3f0323440] error while decoding MB 52 55, bytestream -5 [in#0/rtsp @ 0x55f3f019ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f3f019ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T07:25:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 257.237, "last_started_at": "2026-07-02T07:21:05+09:00", "pid": 47456, "process": {"exists": false, "pid": 47456}, "restart_count": 71, "restart_count_per_hour": 4, "rtsp_timeout_count": 130, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 72}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.159, "sample_count": 2208, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 556061, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:25:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T07:25:23+09:00", "last_frame_read_age_sec": 0.12, "last_frame_read_at": "2026-07-02T07:25:22+09:00", "mjpeg": {"age_sec": 0.112, "exists": true, "mtime": "2026-07-02T07:25:22.962683+09:00", "mtime_epoch": 1782944722.9626834, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 264008}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T07:25:23+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1384.668, "name": "celery", "pid": 267, "rss_mb": 1129.684, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 62, "pushed_frames": 62, "read_frames": 61, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.068, "pushed": 2.068, "read": 2.035}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.202, "count": 62, "max_ms": 7.479, "p95_ms": 7.174}, "gstreamer_push": {"avg_ms": 1.418, "count": 62, "max_ms": 3.846, "p95_ms": 1.5}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.306, "count": 62, "max_ms": 58.299, "p95_ms": 50.715}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.727, "count": 62, "max_ms": 101.525, "p95_ms": 95.949}, "yolo11_640_fire_11class": {"avg_ms": 49.559, "count": 62, "max_ms": 80.691, "p95_ms": 72.519}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.455, "count": 62, "max_ms": 88.128, "p95_ms": 72.064}}, "truncated_count": 0}, "processing_total": {"avg_ms": 78.738, "count": 62, "max_ms": 149.093, "p95_ms": 113.196}, "read_interval": {"avg_ms": 490.535, "count": 61, "max_ms": 15089.743, "p95_ms": 128.977}, "triton_wait": {"avg_ms": 61.74, "count": 62, "max_ms": 101.539, "p95_ms": 95.962}, "visualize": {"avg_ms": 3.765, "count": 62, "max_ms": 5.18, "p95_ms": 5.08}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.975, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 9, "tracking_summary": 4633}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9399.273, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 72, "h264_decode_error_count": 72, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:25:23+09:00", "last_error_summary": "[h264 @ 0x55f3f0323440] error while decoding MB 52 55, bytestream -5 [in#0/rtsp @ 0x55f3f019ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f3f019ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T07:25:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 257.243, "last_started_at": "2026-07-02T07:21:05+09:00", "pid": 47456, "process": {"exists": false, "pid": 47456}, "restart_count": 71, "restart_count_per_hour": 4, "rtsp_timeout_count": 130, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 72}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.159, "sample_count": 2207, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 556061, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:25:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-02T07:25:23+09:00", "last_frame_read_age_sec": 0.127, "last_frame_read_at": "2026-07-02T07:25:22+09:00", "mjpeg": {"age_sec": 0.119, "exists": true, "mtime": "2026-07-02T07:25:22.962683+09:00", "mtime_epoch": 1782944722.9626834, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 264008}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T07:25:23+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1384.668, "name": "celery", "pid": 267, "rss_mb": 1129.684, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 62, "pushed_frames": 62, "read_frames": 61, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.068, "pushed": 2.068, "read": 2.035}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.202, "count": 62, "max_ms": 7.479, "p95_ms": 7.174}, "gstreamer_push": {"avg_ms": 1.418, "count": 62, "max_ms": 3.846, "p95_ms": 1.5}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.306, "count": 62, "max_ms": 58.299, "p95_ms": 50.715}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.727, "count": 62, "max_ms": 101.525, "p95_ms": 95.949}, "yolo11_640_fire_11class": {"avg_ms": 49.559, "count": 62, "max_ms": 80.691, "p95_ms": 72.519}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.455, "count": 62, "max_ms": 88.128, "p95_ms": 72.064}}, "truncated_count": 0}, "processing_total": {"avg_ms": 78.738, "count": 62, "max_ms": 149.093, "p95_ms": 113.196}, "read_interval": {"avg_ms": 490.535, "count": 61, "max_ms": 15089.743, "p95_ms": 128.977}, "triton_wait": {"avg_ms": 61.74, "count": 62, "max_ms": 101.539, "p95_ms": 95.962}, "visualize": {"avg_ms": 3.765, "count": 62, "max_ms": 5.18, "p95_ms": 5.08}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.981, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 9, "tracking_summary": 4633}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9421.082, "limit_mb": 8796093022207.996}, "detail": {"pid": 47715, "restart_count": 72, "start_count": 73}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 72, "h264_decode_error_count": 72, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:25:23+09:00", "last_error_summary": "[h264 @ 0x55f3f0323440] error while decoding MB 52 55, bytestream -5 [in#0/rtsp @ 0x55f3f019ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f3f019ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T07:25:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T07:25:25+09:00", "pid": 47715, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.402, "name": "ffmpeg", "pid": 47715, "rss_mb": 58.402, "state": "S (sleeping)", "threads": 1}, "restart_count": 72, "restart_count_per_hour": 5, "rtsp_timeout_count": 130, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 73}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.159, "sample_count": 2171, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 556061, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:25:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.527, "last_frame_pushed_at": "2026-07-02T07:25:23+09:00", "last_frame_read_age_sec": 4.642, "last_frame_read_at": "2026-07-02T07:25:22+09:00", "mjpeg": {"age_sec": 4.51, "exists": true, "mtime": "2026-07-02T07:25:23.086686+09:00", "mtime_epoch": 1782944723.0866864, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 263990}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T07:25:27+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1384.668, "name": "celery", "pid": 267, "rss_mb": 1129.684, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.617, "pushed": 0.617, "read": 0.617}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.193, "count": 13, "max_ms": 7.089, "p95_ms": 7.089}, "gstreamer_push": {"avg_ms": 1.321, "count": 13, "max_ms": 1.5, "p95_ms": 1.5}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.989, "count": 13, "max_ms": 58.299, "p95_ms": 58.299}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 87.907, "count": 13, "max_ms": 101.525, "p95_ms": 101.525}, "yolo11_640_fire_11class": {"avg_ms": 67.759, "count": 13, "max_ms": 80.691, "p95_ms": 80.691}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.08, "count": 13, "max_ms": 88.128, "p95_ms": 88.128}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.049, "count": 13, "max_ms": 149.093, "p95_ms": 149.093}, "read_interval": {"avg_ms": 1978.063, "count": 13, "max_ms": 15089.743, "p95_ms": 15089.743}, "triton_wait": {"avg_ms": 87.92, "count": 13, "max_ms": 101.539, "p95_ms": 101.539}, "visualize": {"avg_ms": 3.751, "count": 13, "max_ms": 5.105, "p95_ms": 5.105}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.072, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3241}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9476.336, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x560757488700] error while decoding MB 11 20, bytestream -5 [h264 @ 0x560757470f00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x560757470f00] error while decoding MB 0 30, bytestream 50336 [h264 @ 0x56075743a3c0] reference picture missing during reorder [h264 @ 0x56075743a3c0] Missing reference picture, default is 65556 [h264 @ 0x560757e8e040] mmco: unref short failure [h264 @ 0x560757e8e040] illegal short term buffer state detected [h264 @ 0x560757325f00] left block", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 270, "h264_decode_error_count": 476, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:28:57+09:00", "last_error_summary": "[h264 @ 0x560757488700] error while decoding MB 11 20, bytestream -5 [h264 @ 0x560757470f00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x560757470f00] error while decoding MB 0 30, bytestream 50336 [h264 @ 0x56075743a3c0] reference picture missing during reorder [h264 @ 0x56075743a3c0] Missing reference picture, default is 65556 [h264 @ 0x560757e8e040] mmco: unref short failure [h264 @ 0x560757e8e040] illegal short term buffer state detected [h264 @ 0x560757325f00] left block", "last_exit_at": "2026-07-02T07:25:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 231.712, "last_started_at": "2026-07-02T07:25:05+09:00", "pid": 47608, "process": {"exists": false, "pid": 47608}, "restart_count": 302, "restart_count_per_hour": 17, "rtsp_timeout_count": 112, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 303}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.423, "sample_count": 2178, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 388992, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:28:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.065, "last_frame_pushed_at": "2026-07-02T07:28:57+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-02T07:28:57+09:00", "mjpeg": {"age_sec": 0.048, "exists": true, "mtime": "2026-07-02T07:28:57.567428+09:00", "mtime_epoch": 1782944937.567428, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204926}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:28:57+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5543.34, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 62, "pushed_frames": 62, "read_frames": 62, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.068, "pushed": 2.068, "read": 2.068}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 15.173, "count": 62, "max_ms": 20.205, "p95_ms": 17.896}, "gstreamer_push": {"avg_ms": 1.172, "count": 62, "max_ms": 1.503, "p95_ms": 1.318}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.14, "count": 62, "max_ms": 74.845, "p95_ms": 67.21}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.654, "count": 62, "max_ms": 97.777, "p95_ms": 90.633}, "yolo11_640_fire_11class": {"avg_ms": 55.997, "count": 62, "max_ms": 74.136, "p95_ms": 68.782}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.336, "count": 62, "max_ms": 93.121, "p95_ms": 82.983}}, "truncated_count": 0}, "processing_total": {"avg_ms": 118.365, "count": 62, "max_ms": 185.674, "p95_ms": 157.162}, "read_interval": {"avg_ms": 484.435, "count": 62, "max_ms": 12511.478, "p95_ms": 190.095}, "triton_wait": {"avg_ms": 72.667, "count": 62, "max_ms": 97.789, "p95_ms": 90.646}, "visualize": {"avg_ms": 4.403, "count": 62, "max_ms": 6.216, "p95_ms": 6.025}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.979, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3241}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9475.699, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T07:28:57+09:00", "last_error_summary": "[h264 @ 0x560757488700] error while decoding MB 11 20, bytestream -5 [h264 @ 0x560757470f00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x560757470f00] error while decoding MB 0 30, bytestream 50336 [h264 @ 0x56075743a3c0] reference picture missing during reorder [h264 @ 0x56075743a3c0] Missing reference picture, default is 65556 [h264 @ 0x560757e8e040] mmco: unref short failure [h264 @ 0x560757e8e040] illegal short term buffer state detected [h264 @ 0x560757325f00] left block", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 270, "h264_decode_error_count": 476, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:28:57+09:00", "last_error_summary": "[h264 @ 0x560757488700] error while decoding MB 11 20, bytestream -5 [h264 @ 0x560757470f00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x560757470f00] error while decoding MB 0 30, bytestream 50336 [h264 @ 0x56075743a3c0] reference picture missing during reorder [h264 @ 0x56075743a3c0] Missing reference picture, default is 65556 [h264 @ 0x560757e8e040] mmco: unref short failure [h264 @ 0x560757e8e040] illegal short term buffer state detected [h264 @ 0x560757325f00] left block", "last_exit_at": "2026-07-02T07:28:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 231.789, "last_started_at": "2026-07-02T07:25:05+09:00", "pid": 47608, "process": {"exists": false, "pid": 47608}, "restart_count": 302, "restart_count_per_hour": 17, "rtsp_timeout_count": 112, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 303}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.475, "sample_count": 2179, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 388993, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:28:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T07:28:57+09:00", "last_frame_read_age_sec": 0.131, "last_frame_read_at": "2026-07-02T07:28:57+09:00", "mjpeg": {"age_sec": 0.125, "exists": true, "mtime": "2026-07-02T07:28:57.567428+09:00", "mtime_epoch": 1782944937.567428, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204926}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:28:57+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5543.34, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 62, "pushed_frames": 62, "read_frames": 61, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.071, "pushed": 2.071, "read": 2.038}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 15.223, "count": 62, "max_ms": 20.205, "p95_ms": 17.896}, "gstreamer_push": {"avg_ms": 1.166, "count": 62, "max_ms": 1.325, "p95_ms": 1.305}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.862, "count": 62, "max_ms": 74.845, "p95_ms": 67.21}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.254, "count": 62, "max_ms": 97.777, "p95_ms": 90.633}, "yolo11_640_fire_11class": {"avg_ms": 55.683, "count": 62, "max_ms": 74.136, "p95_ms": 68.782}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.983, "count": 62, "max_ms": 93.121, "p95_ms": 82.983}}, "truncated_count": 0}, "processing_total": {"avg_ms": 118.594, "count": 62, "max_ms": 185.674, "p95_ms": 157.162}, "read_interval": {"avg_ms": 490.224, "count": 61, "max_ms": 12511.478, "p95_ms": 190.095}, "triton_wait": {"avg_ms": 72.267, "count": 62, "max_ms": 97.789, "p95_ms": 90.646}, "visualize": {"avg_ms": 4.425, "count": 62, "max_ms": 6.216, "p95_ms": 6.025}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.934, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3241}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9475.809, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 270, "h264_decode_error_count": 476, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:28:57+09:00", "last_error_summary": "[h264 @ 0x560757488700] error while decoding MB 11 20, bytestream -5 [h264 @ 0x560757470f00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x560757470f00] error while decoding MB 0 30, bytestream 50336 [h264 @ 0x56075743a3c0] reference picture missing during reorder [h264 @ 0x56075743a3c0] Missing reference picture, default is 65556 [h264 @ 0x560757e8e040] mmco: unref short failure [h264 @ 0x560757e8e040] illegal short term buffer state detected [h264 @ 0x560757325f00] left block", "last_exit_at": "2026-07-02T07:28:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 231.794, "last_started_at": "2026-07-02T07:25:05+09:00", "pid": 47608, "process": {"exists": false, "pid": 47608}, "restart_count": 302, "restart_count_per_hour": 17, "rtsp_timeout_count": 112, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 303}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.475, "sample_count": 2179, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 388993, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:28:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-02T07:28:57+09:00", "last_frame_read_age_sec": 0.136, "last_frame_read_at": "2026-07-02T07:28:57+09:00", "mjpeg": {"age_sec": 0.13, "exists": true, "mtime": "2026-07-02T07:28:57.567428+09:00", "mtime_epoch": 1782944937.567428, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204926}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:28:57+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5543.34, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 62, "pushed_frames": 62, "read_frames": 61, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.071, "pushed": 2.071, "read": 2.037}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 15.223, "count": 62, "max_ms": 20.205, "p95_ms": 17.896}, "gstreamer_push": {"avg_ms": 1.166, "count": 62, "max_ms": 1.325, "p95_ms": 1.305}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.862, "count": 62, "max_ms": 74.845, "p95_ms": 67.21}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.254, "count": 62, "max_ms": 97.777, "p95_ms": 90.633}, "yolo11_640_fire_11class": {"avg_ms": 55.683, "count": 62, "max_ms": 74.136, "p95_ms": 68.782}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.983, "count": 62, "max_ms": 93.121, "p95_ms": 82.983}}, "truncated_count": 0}, "processing_total": {"avg_ms": 118.594, "count": 62, "max_ms": 185.674, "p95_ms": 157.162}, "read_interval": {"avg_ms": 490.224, "count": 61, "max_ms": 12511.478, "p95_ms": 190.095}, "triton_wait": {"avg_ms": 72.267, "count": 62, "max_ms": 97.789, "p95_ms": 90.646}, "visualize": {"avg_ms": 4.425, "count": 62, "max_ms": 6.216, "p95_ms": 6.025}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.939, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3241}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9584.223, "limit_mb": 8796093022207.996}, "detail": {"pid": 47867, "restart_count": 303, "start_count": 304}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 270, "h264_decode_error_count": 476, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:28:57+09:00", "last_error_summary": "[h264 @ 0x560757488700] error while decoding MB 11 20, bytestream -5 [h264 @ 0x560757470f00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x560757470f00] error while decoding MB 0 30, bytestream 50336 [h264 @ 0x56075743a3c0] reference picture missing during reorder [h264 @ 0x56075743a3c0] Missing reference picture, default is 65556 [h264 @ 0x560757e8e040] mmco: unref short failure [h264 @ 0x560757e8e040] illegal short term buffer state detected [h264 @ 0x560757325f00] left block", "last_exit_at": "2026-07-02T07:29:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T07:29:00+09:00", "pid": 47867, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.051, "name": "ffmpeg", "pid": 47867, "rss_mb": 139.051, "state": "S (sleeping)", "threads": 106}, "restart_count": 303, "restart_count_per_hour": 18, "rtsp_timeout_count": 112, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 304}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.423, "sample_count": 2138, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 388993, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:28:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.72, "last_frame_pushed_at": "2026-07-02T07:28:57+09:00", "last_frame_read_age_sec": 4.846, "last_frame_read_at": "2026-07-02T07:28:57+09:00", "mjpeg": {"age_sec": 4.7, "exists": true, "mtime": "2026-07-02T07:28:57.707431+09:00", "mtime_epoch": 1782944937.707431, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206492}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:29:02+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5543.34, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 25, "pushed_frames": 25, "read_frames": 24, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.835, "pushed": 0.835, "read": 0.802}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 16.157, "count": 24, "max_ms": 20.205, "p95_ms": 19.367}, "gstreamer_push": {"avg_ms": 1.094, "count": 25, "max_ms": 1.302, "p95_ms": 1.29}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.039, "count": 24, "max_ms": 57.135, "p95_ms": 56.322}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.763, "count": 24, "max_ms": 81.979, "p95_ms": 80.029}, "yolo11_640_fire_11class": {"avg_ms": 49.597, "count": 24, "max_ms": 59.085, "p95_ms": 57.646}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.398, "count": 24, "max_ms": 76.689, "p95_ms": 73.867}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.351, "count": 25, "max_ms": 171.041, "p95_ms": 157.162}, "read_interval": {"avg_ms": 1051.885, "count": 24, "max_ms": 12511.478, "p95_ms": 9643.431}, "triton_wait": {"avg_ms": 63.775, "count": 24, "max_ms": 81.993, "p95_ms": 80.04}, "visualize": {"avg_ms": 5.104, "count": 24, "max_ms": 6.216, "p95_ms": 6.049}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.943, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 9, "tracking_summary": 4648}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9408.848, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55b29706ddc0] error while decoding MB 82 66, bytestream -9 [in#0/rtsp @ 0x55b29668ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b29668ae00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 73, "h264_decode_error_count": 73, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:29:19+09:00", "last_error_summary": "[h264 @ 0x55b29706ddc0] error while decoding MB 82 66, bytestream -9 [in#0/rtsp @ 0x55b29668ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b29668ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T07:25:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 234.39, "last_started_at": "2026-07-02T07:25:25+09:00", "pid": 47715, "process": {"exists": false, "pid": 47715}, "restart_count": 72, "restart_count_per_hour": 5, "rtsp_timeout_count": 132, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 73}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 142.419, "sample_count": 2104, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 557789, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:29:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.075, "last_frame_pushed_at": "2026-07-02T07:29:19+09:00", "last_frame_read_age_sec": 0.063, "last_frame_read_at": "2026-07-02T07:29:19+09:00", "mjpeg": {"age_sec": 0.054, "exists": true, "mtime": "2026-07-02T07:29:19.931907+09:00", "mtime_epoch": 1782944959.9319074, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 268956}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T07:29:19+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1384.668, "name": "celery", "pid": 267, "rss_mb": 1138.438, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 33, "pushed_frames": 33, "read_frames": 34, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.1, "pushed": 1.1, "read": 1.134}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.992, "count": 33, "max_ms": 5.558, "p95_ms": 5.347}, "gstreamer_push": {"avg_ms": 1.397, "count": 33, "max_ms": 3.556, "p95_ms": 1.464}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.961, "count": 33, "max_ms": 53.755, "p95_ms": 52.9}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.24, "count": 33, "max_ms": 94.931, "p95_ms": 92.513}, "yolo11_640_fire_11class": {"avg_ms": 49.235, "count": 33, "max_ms": 74.329, "p95_ms": 73.683}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.704, "count": 33, "max_ms": 89.3, "p95_ms": 70.877}}, "truncated_count": 0}, "processing_total": {"avg_ms": 72.043, "count": 33, "max_ms": 108.819, "p95_ms": 106.328}, "read_interval": {"avg_ms": 882.763, "count": 34, "max_ms": 16565.017, "p95_ms": 10643.416}, "triton_wait": {"avg_ms": 58.253, "count": 33, "max_ms": 94.943, "p95_ms": 92.526}, "visualize": {"avg_ms": 3.762, "count": 33, "max_ms": 4.374, "p95_ms": 4.238}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 9, "tracking_summary": 4648}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9408.984, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T07:29:19+09:00", "last_error_summary": "[h264 @ 0x55b29706ddc0] error while decoding MB 82 66, bytestream -9 [in#0/rtsp @ 0x55b29668ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b29668ae00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 73, "h264_decode_error_count": 73, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:29:19+09:00", "last_error_summary": "[h264 @ 0x55b29706ddc0] error while decoding MB 82 66, bytestream -9 [in#0/rtsp @ 0x55b29668ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b29668ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T07:29:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 234.413, "last_started_at": "2026-07-02T07:25:25+09:00", "pid": 47715, "process": {"exists": false, "pid": 47715}, "restart_count": 72, "restart_count_per_hour": 5, "rtsp_timeout_count": 132, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 73}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 142.419, "sample_count": 2105, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 557790, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:29:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-02T07:29:19+09:00", "last_frame_read_age_sec": 0.086, "last_frame_read_at": "2026-07-02T07:29:19+09:00", "mjpeg": {"age_sec": 0.076, "exists": true, "mtime": "2026-07-02T07:29:19.931907+09:00", "mtime_epoch": 1782944959.9319074, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 268956}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T07:29:20+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1384.668, "name": "celery", "pid": 267, "rss_mb": 1138.438, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 34, "pushed_frames": 34, "read_frames": 33, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.135, "pushed": 1.135, "read": 1.102}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.0, "count": 34, "max_ms": 5.558, "p95_ms": 5.347}, "gstreamer_push": {"avg_ms": 1.396, "count": 34, "max_ms": 3.556, "p95_ms": 1.464}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.878, "count": 34, "max_ms": 53.755, "p95_ms": 52.9}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.194, "count": 34, "max_ms": 94.931, "p95_ms": 92.513}, "yolo11_640_fire_11class": {"avg_ms": 49.213, "count": 34, "max_ms": 74.329, "p95_ms": 73.683}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.674, "count": 34, "max_ms": 89.3, "p95_ms": 70.877}}, "truncated_count": 0}, "processing_total": {"avg_ms": 72.12, "count": 34, "max_ms": 108.819, "p95_ms": 106.328}, "read_interval": {"avg_ms": 907.005, "count": 33, "max_ms": 16565.017, "p95_ms": 10643.416}, "triton_wait": {"avg_ms": 58.206, "count": 34, "max_ms": 94.943, "p95_ms": 92.526}, "visualize": {"avg_ms": 3.82, "count": 34, "max_ms": 5.707, "p95_ms": 4.374}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.95, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 9, "tracking_summary": 4648}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9408.477, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 73, "h264_decode_error_count": 73, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:29:19+09:00", "last_error_summary": "[h264 @ 0x55b29706ddc0] error while decoding MB 82 66, bytestream -9 [in#0/rtsp @ 0x55b29668ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b29668ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T07:29:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 234.42, "last_started_at": "2026-07-02T07:25:25+09:00", "pid": 47715, "process": {"exists": false, "pid": 47715}, "restart_count": 72, "restart_count_per_hour": 5, "rtsp_timeout_count": 132, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 73}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 142.419, "sample_count": 2105, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 557790, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:29:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-02T07:29:19+09:00", "last_frame_read_age_sec": 0.093, "last_frame_read_at": "2026-07-02T07:29:19+09:00", "mjpeg": {"age_sec": 0.083, "exists": true, "mtime": "2026-07-02T07:29:19.931907+09:00", "mtime_epoch": 1782944959.9319074, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 268956}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T07:29:20+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1384.668, "name": "celery", "pid": 267, "rss_mb": 1138.438, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 34, "pushed_frames": 34, "read_frames": 33, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.135, "pushed": 1.135, "read": 1.102}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.0, "count": 34, "max_ms": 5.558, "p95_ms": 5.347}, "gstreamer_push": {"avg_ms": 1.396, "count": 34, "max_ms": 3.556, "p95_ms": 1.464}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 39.878, "count": 34, "max_ms": 53.755, "p95_ms": 52.9}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.194, "count": 34, "max_ms": 94.931, "p95_ms": 92.513}, "yolo11_640_fire_11class": {"avg_ms": 49.213, "count": 34, "max_ms": 74.329, "p95_ms": 73.683}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.674, "count": 34, "max_ms": 89.3, "p95_ms": 70.877}}, "truncated_count": 0}, "processing_total": {"avg_ms": 72.12, "count": 34, "max_ms": 108.819, "p95_ms": 106.328}, "read_interval": {"avg_ms": 907.005, "count": 33, "max_ms": 16565.017, "p95_ms": 10643.416}, "triton_wait": {"avg_ms": 58.206, "count": 34, "max_ms": 94.943, "p95_ms": 92.526}, "visualize": {"avg_ms": 3.82, "count": 34, "max_ms": 5.707, "p95_ms": 4.374}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.957, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 9, "tracking_summary": 4648}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9588.18, "limit_mb": 8796093022207.996}, "detail": {"pid": 47974, "restart_count": 73, "start_count": 74}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 73, "h264_decode_error_count": 73, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:29:19+09:00", "last_error_summary": "[h264 @ 0x55b29706ddc0] error while decoding MB 82 66, bytestream -9 [in#0/rtsp @ 0x55b29668ae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55b29668ae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T07:29:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T07:29:22+09:00", "pid": 47974, "process": {"exists": true, "fd_count": 5, "hwm_mb": 210.094, "name": "ffmpeg", "pid": 47974, "rss_mb": 210.094, "state": "S (sleeping)", "threads": 106}, "restart_count": 73, "restart_count_per_hour": 6, "rtsp_timeout_count": 132, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 74}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 142.502, "sample_count": 2067, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 557790, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:29:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.501, "last_frame_pushed_at": "2026-07-02T07:29:19+09:00", "last_frame_read_age_sec": 4.578, "last_frame_read_at": "2026-07-02T07:29:19+09:00", "mjpeg": {"age_sec": 4.481, "exists": true, "mtime": "2026-07-02T07:29:20.019909+09:00", "mtime_epoch": 1782944960.0199094, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 268928}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T07:29:24+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1384.668, "name": "celery", "pid": 267, "rss_mb": 1138.438, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.586, "pushed": 0.586, "read": 0.586}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.033, "count": 13, "max_ms": 5.558, "p95_ms": 5.558}, "gstreamer_push": {"avg_ms": 1.332, "count": 13, "max_ms": 1.428, "p95_ms": 1.428}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.021, "count": 13, "max_ms": 53.755, "p95_ms": 53.755}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.157, "count": 13, "max_ms": 94.931, "p95_ms": 94.931}, "yolo11_640_fire_11class": {"avg_ms": 54.49, "count": 13, "max_ms": 74.329, "p95_ms": 74.329}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.897, "count": 13, "max_ms": 89.3, "p95_ms": 89.3}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.362, "count": 13, "max_ms": 108.819, "p95_ms": 108.819}, "read_interval": {"avg_ms": 2173.489, "count": 13, "max_ms": 16565.017, "p95_ms": 16565.017}, "triton_wait": {"avg_ms": 65.169, "count": 13, "max_ms": 94.943, "p95_ms": 94.943}, "visualize": {"avg_ms": 3.961, "count": 13, "max_ms": 5.707, "p95_ms": 5.707}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.19, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3259}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9472.602, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5626df613fc0] error while decoding MB 28 19, bytestream -30", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 271, "h264_decode_error_count": 477, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:33:30+09:00", "last_error_summary": "[h264 @ 0x5626df613fc0] error while decoding MB 28 19, bytestream -30", "last_exit_at": "2026-07-02T07:29:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 270.419, "last_started_at": "2026-07-02T07:29:00+09:00", "pid": 47867, "process": {"exists": false, "pid": 47867}, "restart_count": 303, "restart_count_per_hour": 16, "rtsp_timeout_count": 112, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 304}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.835, "sample_count": 2218, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 391176, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:33:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.07, "last_frame_pushed_at": "2026-07-02T07:33:30+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-02T07:33:30+09:00", "mjpeg": {"age_sec": 0.056, "exists": true, "mtime": "2026-07-02T07:33:30.769142+09:00", "mtime_epoch": 1782945210.7691417, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200718}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:33:30+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5544.371, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 253, "pushed_frames": 253, "read_frames": 254, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.435, "pushed": 8.435, "read": 8.468}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.146, "count": 253, "max_ms": 22.674, "p95_ms": 15.263}, "gstreamer_push": {"avg_ms": 1.281, "count": 253, "max_ms": 1.391, "p95_ms": 1.36}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.917, "count": 253, "max_ms": 69.955, "p95_ms": 66.263}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.798, "count": 253, "max_ms": 98.495, "p95_ms": 89.383}, "yolo11_640_fire_11class": {"avg_ms": 61.743, "count": 253, "max_ms": 75.619, "p95_ms": 68.706}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.829, "count": 253, "max_ms": 91.305, "p95_ms": 81.819}}, "truncated_count": 0}, "processing_total": {"avg_ms": 102.593, "count": 253, "max_ms": 121.818, "p95_ms": 111.954}, "read_interval": {"avg_ms": 118.359, "count": 254, "max_ms": 140.378, "p95_ms": 129.627}, "triton_wait": {"avg_ms": 79.815, "count": 253, "max_ms": 98.508, "p95_ms": 89.395}, "visualize": {"avg_ms": 3.628, "count": 253, "max_ms": 5.416, "p95_ms": 4.219}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3259}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9472.254, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T07:33:30+09:00", "last_error_summary": "[h264 @ 0x5626df613fc0] error while decoding MB 28 19, bytestream -30", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 271, "h264_decode_error_count": 477, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:33:30+09:00", "last_error_summary": "[h264 @ 0x5626df613fc0] error while decoding MB 28 19, bytestream -30", "last_exit_at": "2026-07-02T07:33:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 270.458, "last_started_at": "2026-07-02T07:29:00+09:00", "pid": 47867, "process": {"exists": false, "pid": 47867}, "restart_count": 303, "restart_count_per_hour": 16, "rtsp_timeout_count": 112, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 304}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.835, "sample_count": 2219, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 391177, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:33:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T07:33:30+09:00", "last_frame_read_age_sec": 0.093, "last_frame_read_at": "2026-07-02T07:33:30+09:00", "mjpeg": {"age_sec": 0.095, "exists": true, "mtime": "2026-07-02T07:33:30.769142+09:00", "mtime_epoch": 1782945210.7691417, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200718}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:33:30+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5544.371, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 254, "pushed_frames": 254, "read_frames": 253, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.481, "pushed": 8.481, "read": 8.448}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.152, "count": 254, "max_ms": 22.674, "p95_ms": 15.358}, "gstreamer_push": {"avg_ms": 1.281, "count": 254, "max_ms": 1.427, "p95_ms": 1.36}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.876, "count": 254, "max_ms": 69.955, "p95_ms": 66.263}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.718, "count": 254, "max_ms": 98.495, "p95_ms": 89.383}, "yolo11_640_fire_11class": {"avg_ms": 61.691, "count": 254, "max_ms": 75.619, "p95_ms": 68.706}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.744, "count": 254, "max_ms": 91.305, "p95_ms": 81.819}}, "truncated_count": 0}, "processing_total": {"avg_ms": 102.52, "count": 254, "max_ms": 121.818, "p95_ms": 111.954}, "read_interval": {"avg_ms": 118.341, "count": 253, "max_ms": 140.378, "p95_ms": 129.627}, "triton_wait": {"avg_ms": 79.735, "count": 254, "max_ms": 98.508, "p95_ms": 89.395}, "visualize": {"avg_ms": 3.629, "count": 254, "max_ms": 5.416, "p95_ms": 4.219}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.95, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3259}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9472.047, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 271, "h264_decode_error_count": 477, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:33:30+09:00", "last_error_summary": "[h264 @ 0x5626df613fc0] error while decoding MB 28 19, bytestream -30", "last_exit_at": "2026-07-02T07:33:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 270.466, "last_started_at": "2026-07-02T07:29:00+09:00", "pid": 47867, "process": {"exists": false, "pid": 47867}, "restart_count": 303, "restart_count_per_hour": 16, "rtsp_timeout_count": 112, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 304}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 137.835, "sample_count": 2219, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 391177, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:33:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-02T07:33:30+09:00", "last_frame_read_age_sec": 0.102, "last_frame_read_at": "2026-07-02T07:33:30+09:00", "mjpeg": {"age_sec": 0.104, "exists": true, "mtime": "2026-07-02T07:33:30.769142+09:00", "mtime_epoch": 1782945210.7691417, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200718}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:33:30+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5544.371, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 254, "pushed_frames": 254, "read_frames": 253, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.479, "pushed": 8.479, "read": 8.445}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.152, "count": 254, "max_ms": 22.674, "p95_ms": 15.358}, "gstreamer_push": {"avg_ms": 1.281, "count": 254, "max_ms": 1.427, "p95_ms": 1.36}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.876, "count": 254, "max_ms": 69.955, "p95_ms": 66.263}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.718, "count": 254, "max_ms": 98.495, "p95_ms": 89.383}, "yolo11_640_fire_11class": {"avg_ms": 61.691, "count": 254, "max_ms": 75.619, "p95_ms": 68.706}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.744, "count": 254, "max_ms": 91.305, "p95_ms": 81.819}}, "truncated_count": 0}, "processing_total": {"avg_ms": 102.52, "count": 254, "max_ms": 121.818, "p95_ms": 111.954}, "read_interval": {"avg_ms": 118.341, "count": 253, "max_ms": 140.378, "p95_ms": 129.627}, "triton_wait": {"avg_ms": 79.735, "count": 254, "max_ms": 98.508, "p95_ms": 89.395}, "visualize": {"avg_ms": 3.629, "count": 254, "max_ms": 5.416, "p95_ms": 4.219}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.958, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3259}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9577.66, "limit_mb": 8796093022207.996}, "detail": {"pid": 48126, "restart_count": 304, "start_count": 305}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 271, "h264_decode_error_count": 477, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:33:30+09:00", "last_error_summary": "[h264 @ 0x5626df613fc0] error while decoding MB 28 19, bytestream -30", "last_exit_at": "2026-07-02T07:33:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T07:33:32+09:00", "pid": 48126, "process": {"exists": true, "fd_count": 5, "hwm_mb": 137.352, "name": "ffmpeg", "pid": 48126, "rss_mb": 137.352, "state": "S (sleeping)", "threads": 106}, "restart_count": 304, "restart_count_per_hour": 17, "rtsp_timeout_count": 112, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 305}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 126.731, "sample_count": 2198, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 391177, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:33:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.13, "last_frame_pushed_at": "2026-07-02T07:33:30+09:00", "last_frame_read_age_sec": 4.216, "last_frame_read_at": "2026-07-02T07:33:30+09:00", "mjpeg": {"age_sec": 4.114, "exists": true, "mtime": "2026-07-02T07:33:30.873144+09:00", "mtime_epoch": 1782945210.8731437, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 201128}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:33:34+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5544.371, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 217, "pushed_frames": 217, "read_frames": 216, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.25, "pushed": 7.25, "read": 7.217}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.145, "count": 217, "max_ms": 22.674, "p95_ms": 15.021}, "gstreamer_push": {"avg_ms": 1.28, "count": 217, "max_ms": 1.427, "p95_ms": 1.353}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.84, "count": 217, "max_ms": 69.955, "p95_ms": 66.419}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.882, "count": 217, "max_ms": 98.495, "p95_ms": 89.383}, "yolo11_640_fire_11class": {"avg_ms": 62.249, "count": 217, "max_ms": 75.619, "p95_ms": 68.706}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.814, "count": 217, "max_ms": 87.336, "p95_ms": 82.006}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.666, "count": 217, "max_ms": 121.818, "p95_ms": 111.954}, "read_interval": {"avg_ms": 119.541, "count": 216, "max_ms": 140.378, "p95_ms": 129.854}, "triton_wait": {"avg_ms": 80.895, "count": 217, "max_ms": 98.508, "p95_ms": 89.395}, "visualize": {"avg_ms": 3.631, "count": 217, "max_ms": 5.416, "p95_ms": 4.255}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.93, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 9, "tracking_summary": 4665}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9406.301, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x559f03ae2880] error while decoding MB 102 31, bytestream -13 [in#0/rtsp @ 0x559f0398ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559f0398ce00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 74, "h264_decode_error_count": 74, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:33:56+09:00", "last_error_summary": "[h264 @ 0x559f03ae2880] error while decoding MB 102 31, bytestream -13 [in#0/rtsp @ 0x559f0398ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559f0398ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T07:29:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 274.14, "last_started_at": "2026-07-02T07:29:22+09:00", "pid": 47974, "process": {"exists": false, "pid": 47974}, "restart_count": 73, "restart_count_per_hour": 6, "rtsp_timeout_count": 134, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 74}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.792, "sample_count": 2091, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 559868, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:33:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.081, "last_frame_pushed_at": "2026-07-02T07:33:56+09:00", "last_frame_read_age_sec": 0.071, "last_frame_read_at": "2026-07-02T07:33:56+09:00", "mjpeg": {"age_sec": 0.064, "exists": true, "mtime": "2026-07-02T07:33:56.577668+09:00", "mtime_epoch": 1782945236.577668, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 265792}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T07:33:56+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1384.668, "name": "celery", "pid": 267, "rss_mb": 1132.348, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 41, "pushed_frames": 41, "read_frames": 41, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.369, "pushed": 1.369, "read": 1.369}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.793, "count": 41, "max_ms": 8.495, "p95_ms": 7.568}, "gstreamer_push": {"avg_ms": 1.733, "count": 41, "max_ms": 7.0, "p95_ms": 3.988}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.941, "count": 41, "max_ms": 67.126, "p95_ms": 61.342}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.107, "count": 41, "max_ms": 97.966, "p95_ms": 95.993}, "yolo11_640_fire_11class": {"avg_ms": 63.513, "count": 41, "max_ms": 75.224, "p95_ms": 75.172}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.115, "count": 41, "max_ms": 90.995, "p95_ms": 88.755}}, "truncated_count": 0}, "processing_total": {"avg_ms": 102.187, "count": 41, "max_ms": 152.326, "p95_ms": 119.59}, "read_interval": {"avg_ms": 730.739, "count": 41, "max_ms": 15164.355, "p95_ms": 163.884}, "triton_wait": {"avg_ms": 81.286, "count": 41, "max_ms": 97.978, "p95_ms": 96.007}, "visualize": {"avg_ms": 4.004, "count": 41, "max_ms": 5.152, "p95_ms": 5.086}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.938, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 9, "tracking_summary": 4665}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9408.09, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T07:33:56+09:00", "last_error_summary": "[h264 @ 0x559f03ae2880] error while decoding MB 102 31, bytestream -13 [in#0/rtsp @ 0x559f0398ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559f0398ce00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 74, "h264_decode_error_count": 74, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:33:56+09:00", "last_error_summary": "[h264 @ 0x559f03ae2880] error while decoding MB 102 31, bytestream -13 [in#0/rtsp @ 0x559f0398ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559f0398ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T07:33:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 274.212, "last_started_at": "2026-07-02T07:29:22+09:00", "pid": 47974, "process": {"exists": false, "pid": 47974}, "restart_count": 73, "restart_count_per_hour": 6, "rtsp_timeout_count": 134, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 74}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.792, "sample_count": 2092, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 559869, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:33:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T07:33:56+09:00", "last_frame_read_age_sec": 0.143, "last_frame_read_at": "2026-07-02T07:33:56+09:00", "mjpeg": {"age_sec": 0.134, "exists": true, "mtime": "2026-07-02T07:33:56.577668+09:00", "mtime_epoch": 1782945236.577668, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 265792}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T07:33:56+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1384.668, "name": "celery", "pid": 267, "rss_mb": 1134.926, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 41, "pushed_frames": 41, "read_frames": 41, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.367, "pushed": 1.367, "read": 1.367}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.949, "count": 41, "max_ms": 11.499, "p95_ms": 7.598}, "gstreamer_push": {"avg_ms": 1.73, "count": 41, "max_ms": 7.0, "p95_ms": 3.988}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.903, "count": 41, "max_ms": 67.126, "p95_ms": 61.342}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.709, "count": 41, "max_ms": 97.966, "p95_ms": 95.993}, "yolo11_640_fire_11class": {"avg_ms": 63.346, "count": 41, "max_ms": 75.224, "p95_ms": 75.172}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.926, "count": 41, "max_ms": 90.995, "p95_ms": 88.755}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.199, "count": 41, "max_ms": 152.326, "p95_ms": 137.29}, "read_interval": {"avg_ms": 730.739, "count": 41, "max_ms": 15164.355, "p95_ms": 163.884}, "triton_wait": {"avg_ms": 80.888, "count": 41, "max_ms": 97.978, "p95_ms": 96.007}, "visualize": {"avg_ms": 4.076, "count": 41, "max_ms": 6.517, "p95_ms": 5.138}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 9, "tracking_summary": 4665}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9408.074, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 74, "h264_decode_error_count": 74, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:33:56+09:00", "last_error_summary": "[h264 @ 0x559f03ae2880] error while decoding MB 102 31, bytestream -13 [in#0/rtsp @ 0x559f0398ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559f0398ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T07:33:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 274.217, "last_started_at": "2026-07-02T07:29:22+09:00", "pid": 47974, "process": {"exists": false, "pid": 47974}, "restart_count": 73, "restart_count_per_hour": 6, "rtsp_timeout_count": 134, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 74}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.792, "sample_count": 2092, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 559869, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:33:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-02T07:33:56+09:00", "last_frame_read_age_sec": 0.148, "last_frame_read_at": "2026-07-02T07:33:56+09:00", "mjpeg": {"age_sec": 0.139, "exists": true, "mtime": "2026-07-02T07:33:56.577668+09:00", "mtime_epoch": 1782945236.577668, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 265792}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T07:33:56+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1384.668, "name": "celery", "pid": 267, "rss_mb": 1134.926, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 41, "pushed_frames": 41, "read_frames": 41, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.367, "pushed": 1.367, "read": 1.367}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.949, "count": 41, "max_ms": 11.499, "p95_ms": 7.598}, "gstreamer_push": {"avg_ms": 1.73, "count": 41, "max_ms": 7.0, "p95_ms": 3.988}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.903, "count": 41, "max_ms": 67.126, "p95_ms": 61.342}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.709, "count": 41, "max_ms": 97.966, "p95_ms": 95.993}, "yolo11_640_fire_11class": {"avg_ms": 63.346, "count": 41, "max_ms": 75.224, "p95_ms": 75.172}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.926, "count": 41, "max_ms": 90.995, "p95_ms": 88.755}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.199, "count": 41, "max_ms": 152.326, "p95_ms": 137.29}, "read_interval": {"avg_ms": 730.739, "count": 41, "max_ms": 15164.355, "p95_ms": 163.884}, "triton_wait": {"avg_ms": 80.888, "count": 41, "max_ms": 97.978, "p95_ms": 96.007}, "visualize": {"avg_ms": 4.076, "count": 41, "max_ms": 6.517, "p95_ms": 5.138}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 9, "tracking_summary": 4665}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9430.281, "limit_mb": 8796093022207.996}, "detail": {"pid": 48233, "restart_count": 74, "start_count": 75}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 74, "h264_decode_error_count": 74, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:33:56+09:00", "last_error_summary": "[h264 @ 0x559f03ae2880] error while decoding MB 102 31, bytestream -13 [in#0/rtsp @ 0x559f0398ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x559f0398ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T07:33:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T07:33:58+09:00", "pid": 48233, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.531, "name": "ffmpeg", "pid": 48233, "rss_mb": 58.531, "state": "S (sleeping)", "threads": 1}, "restart_count": 74, "restart_count_per_hour": 7, "rtsp_timeout_count": 134, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 75}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.792, "sample_count": 2092, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 559869, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:33:56+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.112, "last_frame_pushed_at": "2026-07-02T07:33:56+09:00", "last_frame_read_age_sec": 4.251, "last_frame_read_at": "2026-07-02T07:33:56+09:00", "mjpeg": {"age_sec": 4.086, "exists": true, "mtime": "2026-07-02T07:33:56.733671+09:00", "mtime_epoch": 1782945236.7336712, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 265689}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T07:34:00+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1384.668, "name": "celery", "pid": 267, "rss_mb": 1134.926, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.825, "pushed": 0.825, "read": 0.825}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.586, "count": 13, "max_ms": 11.499, "p95_ms": 11.499}, "gstreamer_push": {"avg_ms": 1.25, "count": 13, "max_ms": 1.432, "p95_ms": 1.432}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.352, "count": 13, "max_ms": 61.342, "p95_ms": 61.342}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.476, "count": 13, "max_ms": 92.961, "p95_ms": 92.961}, "yolo11_640_fire_11class": {"avg_ms": 55.798, "count": 13, "max_ms": 70.762, "p95_ms": 70.762}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.333, "count": 13, "max_ms": 84.006, "p95_ms": 84.006}}, "truncated_count": 0}, "processing_total": {"avg_ms": 108.614, "count": 13, "max_ms": 152.326, "p95_ms": 152.326}, "read_interval": {"avg_ms": 2051.712, "count": 13, "max_ms": 15164.355, "p95_ms": 15164.355}, "triton_wait": {"avg_ms": 70.49, "count": 13, "max_ms": 92.974, "p95_ms": 92.974}, "visualize": {"avg_ms": 4.533, "count": 13, "max_ms": 6.517, "p95_ms": 6.517}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 15.759, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3283}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9478.867, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 9, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5573ef74a500] cabac decode of qscale diff failed at 40 30 [h264 @ 0x5573ef74a500] error while decoding MB 40 30, bytestream 127268 [h264 @ 0x5573ef722180] cabac decode of qscale diff failed at 26 43 [h264 @ 0x5573ef722180] error while decoding MB 26 43, bytestream 125797 [h264 @ 0x5573ef63c600] left block unavailable for requested intra mode [h264 @ 0x5573ef63c600] error while decoding MB 0 27, bytestream 29306 [h264 @ 0x5573ef63c600] error while decoding MB 23 9, bytestream -5 [h264 @", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 272, "h264_decode_error_count": 486, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:39:33+09:00", "last_error_summary": "[h264 @ 0x5573ef74a500] cabac decode of qscale diff failed at 40 30 [h264 @ 0x5573ef74a500] error while decoding MB 40 30, bytestream 127268 [h264 @ 0x5573ef722180] cabac decode of qscale diff failed at 26 43 [h264 @ 0x5573ef722180] error while decoding MB 26 43, bytestream 125797 [h264 @ 0x5573ef63c600] left block unavailable for requested intra mode [h264 @ 0x5573ef63c600] error while decoding MB 0 27, bytestream 29306 [h264 @ 0x5573ef63c600] error while decoding MB 23 9, bytestream -5 [h264 @", "last_exit_at": "2026-07-02T07:33:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 360.613, "last_started_at": "2026-07-02T07:33:32+09:00", "pid": 48126, "process": {"exists": false, "pid": 48126}, "restart_count": 304, "restart_count_per_hour": 15, "rtsp_timeout_count": 114, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 305}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 126.751, "sample_count": 2335, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 394055, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:39:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.068, "last_frame_pushed_at": "2026-07-02T07:39:33+09:00", "last_frame_read_age_sec": 0.056, "last_frame_read_at": "2026-07-02T07:39:33+09:00", "mjpeg": {"age_sec": 0.051, "exists": true, "mtime": "2026-07-02T07:39:33.548399+09:00", "mtime_epoch": 1782945573.5483994, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 210875}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:39:33+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5545.918, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 81, "pushed_frames": 81, "read_frames": 81, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.701, "pushed": 2.701, "read": 2.701}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 13.783, "count": 81, "max_ms": 14.968, "p95_ms": 14.682}, "gstreamer_push": {"avg_ms": 1.246, "count": 81, "max_ms": 2.255, "p95_ms": 1.337}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.345, "count": 81, "max_ms": 68.608, "p95_ms": 67.223}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.874, "count": 81, "max_ms": 94.433, "p95_ms": 89.481}, "yolo11_640_fire_11class": {"avg_ms": 63.247, "count": 81, "max_ms": 73.975, "p95_ms": 68.442}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.499, "count": 81, "max_ms": 84.153, "p95_ms": 81.886}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.951, "count": 81, "max_ms": 116.434, "p95_ms": 111.81}, "read_interval": {"avg_ms": 370.688, "count": 81, "max_ms": 20496.128, "p95_ms": 127.217}, "triton_wait": {"avg_ms": 81.887, "count": 81, "max_ms": 94.445, "p95_ms": 89.493}, "visualize": {"avg_ms": 3.5, "count": 81, "max_ms": 3.935, "p95_ms": 3.723}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.983, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3283}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9478.141, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T07:39:33+09:00", "last_error_summary": "[h264 @ 0x5573ef74a500] cabac decode of qscale diff failed at 40 30 [h264 @ 0x5573ef74a500] error while decoding MB 40 30, bytestream 127268 [h264 @ 0x5573ef722180] cabac decode of qscale diff failed at 26 43 [h264 @ 0x5573ef722180] error while decoding MB 26 43, bytestream 125797 [h264 @ 0x5573ef63c600] left block unavailable for requested intra mode [h264 @ 0x5573ef63c600] error while decoding MB 0 27, bytestream 29306 [h264 @ 0x5573ef63c600] error while decoding MB 23 9, bytestream -5 [h264 @", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 272, "h264_decode_error_count": 486, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:39:33+09:00", "last_error_summary": "[h264 @ 0x5573ef74a500] cabac decode of qscale diff failed at 40 30 [h264 @ 0x5573ef74a500] error while decoding MB 40 30, bytestream 127268 [h264 @ 0x5573ef722180] cabac decode of qscale diff failed at 26 43 [h264 @ 0x5573ef722180] error while decoding MB 26 43, bytestream 125797 [h264 @ 0x5573ef63c600] left block unavailable for requested intra mode [h264 @ 0x5573ef63c600] error while decoding MB 0 27, bytestream 29306 [h264 @ 0x5573ef63c600] error while decoding MB 23 9, bytestream -5 [h264 @", "last_exit_at": "2026-07-02T07:39:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 360.678, "last_started_at": "2026-07-02T07:33:32+09:00", "pid": 48126, "process": {"exists": false, "pid": 48126}, "restart_count": 304, "restart_count_per_hour": 15, "rtsp_timeout_count": 114, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 305}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 126.751, "sample_count": 2336, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 394056, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:39:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-02T07:39:33+09:00", "last_frame_read_age_sec": 0.121, "last_frame_read_at": "2026-07-02T07:39:33+09:00", "mjpeg": {"age_sec": 0.116, "exists": true, "mtime": "2026-07-02T07:39:33.548399+09:00", "mtime_epoch": 1782945573.5483994, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 210875}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:39:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5545.918, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 81, "pushed_frames": 81, "read_frames": 80, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.707, "pushed": 2.707, "read": 2.674}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 13.786, "count": 81, "max_ms": 14.968, "p95_ms": 14.682}, "gstreamer_push": {"avg_ms": 1.244, "count": 81, "max_ms": 2.255, "p95_ms": 1.328}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.672, "count": 81, "max_ms": 68.608, "p95_ms": 67.45}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.026, "count": 81, "max_ms": 94.433, "p95_ms": 89.481}, "yolo11_640_fire_11class": {"avg_ms": 63.384, "count": 81, "max_ms": 73.975, "p95_ms": 68.442}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.735, "count": 81, "max_ms": 84.153, "p95_ms": 81.888}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.108, "count": 81, "max_ms": 116.434, "p95_ms": 111.81}, "read_interval": {"avg_ms": 373.903, "count": 80, "max_ms": 20496.128, "p95_ms": 127.217}, "triton_wait": {"avg_ms": 82.039, "count": 81, "max_ms": 94.445, "p95_ms": 89.493}, "visualize": {"avg_ms": 3.499, "count": 81, "max_ms": 3.935, "p95_ms": 3.723}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.917, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3283}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9478.23, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 272, "h264_decode_error_count": 486, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:39:33+09:00", "last_error_summary": "[h264 @ 0x5573ef74a500] cabac decode of qscale diff failed at 40 30 [h264 @ 0x5573ef74a500] error while decoding MB 40 30, bytestream 127268 [h264 @ 0x5573ef722180] cabac decode of qscale diff failed at 26 43 [h264 @ 0x5573ef722180] error while decoding MB 26 43, bytestream 125797 [h264 @ 0x5573ef63c600] left block unavailable for requested intra mode [h264 @ 0x5573ef63c600] error while decoding MB 0 27, bytestream 29306 [h264 @ 0x5573ef63c600] error while decoding MB 23 9, bytestream -5 [h264 @", "last_exit_at": "2026-07-02T07:39:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 360.685, "last_started_at": "2026-07-02T07:33:32+09:00", "pid": 48126, "process": {"exists": false, "pid": 48126}, "restart_count": 304, "restart_count_per_hour": 15, "rtsp_timeout_count": 114, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 305}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 126.751, "sample_count": 2336, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 394056, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:39:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-02T07:39:33+09:00", "last_frame_read_age_sec": 0.128, "last_frame_read_at": "2026-07-02T07:39:33+09:00", "mjpeg": {"age_sec": 0.122, "exists": true, "mtime": "2026-07-02T07:39:33.548399+09:00", "mtime_epoch": 1782945573.5483994, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 210875}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:39:33+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5545.918, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 81, "pushed_frames": 81, "read_frames": 80, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.707, "pushed": 2.707, "read": 2.673}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 13.786, "count": 81, "max_ms": 14.968, "p95_ms": 14.682}, "gstreamer_push": {"avg_ms": 1.244, "count": 81, "max_ms": 2.255, "p95_ms": 1.328}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.672, "count": 81, "max_ms": 68.608, "p95_ms": 67.45}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.026, "count": 81, "max_ms": 94.433, "p95_ms": 89.481}, "yolo11_640_fire_11class": {"avg_ms": 63.384, "count": 81, "max_ms": 73.975, "p95_ms": 68.442}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.735, "count": 81, "max_ms": 84.153, "p95_ms": 81.888}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.108, "count": 81, "max_ms": 116.434, "p95_ms": 111.81}, "read_interval": {"avg_ms": 373.903, "count": 80, "max_ms": 20496.128, "p95_ms": 127.217}, "triton_wait": {"avg_ms": 82.039, "count": 81, "max_ms": 94.445, "p95_ms": 89.493}, "visualize": {"avg_ms": 3.499, "count": 81, "max_ms": 3.935, "p95_ms": 3.723}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.923, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3283}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9583.23, "limit_mb": 8796093022207.996}, "detail": {"pid": 48385, "restart_count": 305, "start_count": 306}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 272, "h264_decode_error_count": 486, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:39:33+09:00", "last_error_summary": "[h264 @ 0x5573ef74a500] cabac decode of qscale diff failed at 40 30 [h264 @ 0x5573ef74a500] error while decoding MB 40 30, bytestream 127268 [h264 @ 0x5573ef722180] cabac decode of qscale diff failed at 26 43 [h264 @ 0x5573ef722180] error while decoding MB 26 43, bytestream 125797 [h264 @ 0x5573ef63c600] left block unavailable for requested intra mode [h264 @ 0x5573ef63c600] error while decoding MB 0 27, bytestream 29306 [h264 @ 0x5573ef63c600] error while decoding MB 23 9, bytestream -5 [h264 @", "last_exit_at": "2026-07-02T07:39:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T07:39:35+09:00", "pid": 48385, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.129, "name": "ffmpeg", "pid": 48385, "rss_mb": 136.129, "state": "S (sleeping)", "threads": 106}, "restart_count": 305, "restart_count_per_hour": 16, "rtsp_timeout_count": 114, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 306}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 126.926, "sample_count": 2303, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 394056, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:39:33+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.091, "last_frame_pushed_at": "2026-07-02T07:39:33+09:00", "last_frame_read_age_sec": 4.204, "last_frame_read_at": "2026-07-02T07:39:33+09:00", "mjpeg": {"age_sec": 4.075, "exists": true, "mtime": "2026-07-02T07:39:33.672402+09:00", "mtime_epoch": 1782945573.672402, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 210772}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:39:37+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5545.918, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 46, "pushed_frames": 46, "read_frames": 45, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.536, "pushed": 1.536, "read": 1.503}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 13.748, "count": 46, "max_ms": 14.682, "p95_ms": 14.15}, "gstreamer_push": {"avg_ms": 1.243, "count": 46, "max_ms": 2.255, "p95_ms": 1.308}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.433, "count": 46, "max_ms": 68.608, "p95_ms": 67.728}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 84.681, "count": 46, "max_ms": 91.39, "p95_ms": 89.481}, "yolo11_640_fire_11class": {"avg_ms": 65.093, "count": 46, "max_ms": 70.743, "p95_ms": 69.125}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.432, "count": 46, "max_ms": 84.153, "p95_ms": 82.189}}, "truncated_count": 0}, "processing_total": {"avg_ms": 106.663, "count": 46, "max_ms": 113.711, "p95_ms": 111.81}, "read_interval": {"avg_ms": 574.381, "count": 45, "max_ms": 20496.128, "p95_ms": 127.217}, "triton_wait": {"avg_ms": 84.693, "count": 46, "max_ms": 91.402, "p95_ms": 89.493}, "visualize": {"avg_ms": 3.473, "count": 46, "max_ms": 3.904, "p95_ms": 3.723}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.947, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3308}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9483.117, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5593ec535dc0] cabac decode of qscale diff failed at 13 32 [h264 @ 0x5593ec535dc0] error while decoding MB 13 32, bytestream 2611 [h264 @ 0x5593ec42c580] error while decoding MB 68 18, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 273, "h264_decode_error_count": 489, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:45:25+09:00", "last_error_summary": "[h264 @ 0x5593ec535dc0] cabac decode of qscale diff failed at 13 32 [h264 @ 0x5593ec535dc0] error while decoding MB 13 32, bytestream 2611 [h264 @ 0x5593ec42c580] error while decoding MB 68 18, bytestream -7", "last_exit_at": "2026-07-02T07:39:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 349.985, "last_started_at": "2026-07-02T07:39:35+09:00", "pid": 48385, "process": {"exists": false, "pid": 48385}, "restart_count": 305, "restart_count_per_hour": 14, "rtsp_timeout_count": 114, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 306}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 123.979, "sample_count": 2505, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 397012, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:45:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.067, "last_frame_pushed_at": "2026-07-02T07:45:25+09:00", "last_frame_read_age_sec": 0.057, "last_frame_read_at": "2026-07-02T07:45:25+09:00", "mjpeg": {"age_sec": 0.048, "exists": true, "mtime": "2026-07-02T07:45:25.683254+09:00", "mtime_epoch": 1782945925.6832545, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172008}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:45:25+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5547.207, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 227, "pushed_frames": 227, "read_frames": 227, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.594, "pushed": 7.594, "read": 7.594}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 16.081, "count": 227, "max_ms": 24.423, "p95_ms": 18.339}, "gstreamer_push": {"avg_ms": 1.129, "count": 227, "max_ms": 1.496, "p95_ms": 1.325}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.134, "count": 227, "max_ms": 75.459, "p95_ms": 58.442}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.546, "count": 227, "max_ms": 97.998, "p95_ms": 91.449}, "yolo11_640_fire_11class": {"avg_ms": 54.059, "count": 227, "max_ms": 79.941, "p95_ms": 73.366}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.611, "count": 227, "max_ms": 98.691, "p95_ms": 87.422}}, "truncated_count": 0}, "processing_total": {"avg_ms": 118.399, "count": 227, "max_ms": 159.607, "p95_ms": 150.758}, "read_interval": {"avg_ms": 131.936, "count": 227, "max_ms": 174.203, "p95_ms": 163.887}, "triton_wait": {"avg_ms": 65.562, "count": 227, "max_ms": 98.703, "p95_ms": 91.461}, "visualize": {"avg_ms": 4.591, "count": 227, "max_ms": 5.956, "p95_ms": 5.197}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.891, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3308}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9482.672, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T07:45:25+09:00", "last_error_summary": "[h264 @ 0x5593ec535dc0] cabac decode of qscale diff failed at 13 32 [h264 @ 0x5593ec535dc0] error while decoding MB 13 32, bytestream 2611 [h264 @ 0x5593ec42c580] error while decoding MB 68 18, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 273, "h264_decode_error_count": 489, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:45:25+09:00", "last_error_summary": "[h264 @ 0x5593ec535dc0] cabac decode of qscale diff failed at 13 32 [h264 @ 0x5593ec535dc0] error while decoding MB 13 32, bytestream 2611 [h264 @ 0x5593ec42c580] error while decoding MB 68 18, bytestream -7", "last_exit_at": "2026-07-02T07:45:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 350.071, "last_started_at": "2026-07-02T07:39:35+09:00", "pid": 48385, "process": {"exists": false, "pid": 48385}, "restart_count": 305, "restart_count_per_hour": 14, "rtsp_timeout_count": 114, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 306}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 124.069, "sample_count": 2505, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 397013, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:45:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T07:45:25+09:00", "last_frame_read_age_sec": 0.142, "last_frame_read_at": "2026-07-02T07:45:25+09:00", "mjpeg": {"age_sec": 0.134, "exists": true, "mtime": "2026-07-02T07:45:25.683254+09:00", "mtime_epoch": 1782945925.6832545, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172008}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:45:25+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5547.207, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 228, "pushed_frames": 228, "read_frames": 227, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.606, "pushed": 7.606, "read": 7.573}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 16.093, "count": 228, "max_ms": 24.423, "p95_ms": 18.459}, "gstreamer_push": {"avg_ms": 1.129, "count": 228, "max_ms": 1.496, "p95_ms": 1.325}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.101, "count": 228, "max_ms": 75.459, "p95_ms": 58.442}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.54, "count": 228, "max_ms": 97.998, "p95_ms": 91.449}, "yolo11_640_fire_11class": {"avg_ms": 54.018, "count": 228, "max_ms": 79.941, "p95_ms": 73.366}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.628, "count": 228, "max_ms": 98.691, "p95_ms": 87.422}}, "truncated_count": 0}, "processing_total": {"avg_ms": 118.47, "count": 228, "max_ms": 159.607, "p95_ms": 150.758}, "read_interval": {"avg_ms": 131.936, "count": 227, "max_ms": 174.203, "p95_ms": 163.887}, "triton_wait": {"avg_ms": 65.556, "count": 228, "max_ms": 98.703, "p95_ms": 91.461}, "visualize": {"avg_ms": 4.599, "count": 228, "max_ms": 6.426, "p95_ms": 5.209}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.976, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3308}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9482.664, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 273, "h264_decode_error_count": 489, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:45:25+09:00", "last_error_summary": "[h264 @ 0x5593ec535dc0] cabac decode of qscale diff failed at 13 32 [h264 @ 0x5593ec535dc0] error while decoding MB 13 32, bytestream 2611 [h264 @ 0x5593ec42c580] error while decoding MB 68 18, bytestream -7", "last_exit_at": "2026-07-02T07:45:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 350.077, "last_started_at": "2026-07-02T07:39:35+09:00", "pid": 48385, "process": {"exists": false, "pid": 48385}, "restart_count": 305, "restart_count_per_hour": 14, "rtsp_timeout_count": 114, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 306}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 124.069, "sample_count": 2505, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 397013, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:45:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-02T07:45:25+09:00", "last_frame_read_age_sec": 0.149, "last_frame_read_at": "2026-07-02T07:45:25+09:00", "mjpeg": {"age_sec": 0.14, "exists": true, "mtime": "2026-07-02T07:45:25.683254+09:00", "mtime_epoch": 1782945925.6832545, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 172008}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:45:25+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5547.207, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 228, "pushed_frames": 228, "read_frames": 227, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.604, "pushed": 7.604, "read": 7.571}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 16.093, "count": 228, "max_ms": 24.423, "p95_ms": 18.459}, "gstreamer_push": {"avg_ms": 1.129, "count": 228, "max_ms": 1.496, "p95_ms": 1.325}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.101, "count": 228, "max_ms": 75.459, "p95_ms": 58.442}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.54, "count": 228, "max_ms": 97.998, "p95_ms": 91.449}, "yolo11_640_fire_11class": {"avg_ms": 54.018, "count": 228, "max_ms": 79.941, "p95_ms": 73.366}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.628, "count": 228, "max_ms": 98.691, "p95_ms": 87.422}}, "truncated_count": 0}, "processing_total": {"avg_ms": 118.47, "count": 228, "max_ms": 159.607, "p95_ms": 150.758}, "read_interval": {"avg_ms": 131.936, "count": 227, "max_ms": 174.203, "p95_ms": 163.887}, "triton_wait": {"avg_ms": 65.556, "count": 228, "max_ms": 98.703, "p95_ms": 91.461}, "visualize": {"avg_ms": 4.599, "count": 228, "max_ms": 6.426, "p95_ms": 5.209}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.982, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3308}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9587.105, "limit_mb": 8796093022207.996}, "detail": {"pid": 48492, "restart_count": 306, "start_count": 307}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 273, "h264_decode_error_count": 489, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:45:25+09:00", "last_error_summary": "[h264 @ 0x5593ec535dc0] cabac decode of qscale diff failed at 13 32 [h264 @ 0x5593ec535dc0] error while decoding MB 13 32, bytestream 2611 [h264 @ 0x5593ec42c580] error while decoding MB 68 18, bytestream -7", "last_exit_at": "2026-07-02T07:45:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T07:45:28+09:00", "pid": 48492, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.336, "name": "ffmpeg", "pid": 48492, "rss_mb": 141.336, "state": "S (sleeping)", "threads": 106}, "restart_count": 306, "restart_count_per_hour": 15, "rtsp_timeout_count": 114, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 307}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 124.142, "sample_count": 2469, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 397013, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:45:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.519, "last_frame_pushed_at": "2026-07-02T07:45:25+09:00", "last_frame_read_age_sec": 4.655, "last_frame_read_at": "2026-07-02T07:45:25+09:00", "mjpeg": {"age_sec": 4.499, "exists": true, "mtime": "2026-07-02T07:45:25.831257+09:00", "mtime_epoch": 1782945925.8312573, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171823}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:45:30+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5547.207, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 186, "pushed_frames": 186, "read_frames": 185, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.211, "pushed": 6.211, "read": 6.178}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 16.284, "count": 186, "max_ms": 20.044, "p95_ms": 18.223}, "gstreamer_push": {"avg_ms": 1.099, "count": 186, "max_ms": 1.496, "p95_ms": 1.214}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.206, "count": 186, "max_ms": 75.459, "p95_ms": 58.608}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.401, "count": 186, "max_ms": 97.998, "p95_ms": 91.311}, "yolo11_640_fire_11class": {"avg_ms": 53.903, "count": 186, "max_ms": 79.941, "p95_ms": 72.492}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.666, "count": 186, "max_ms": 98.691, "p95_ms": 88.163}}, "truncated_count": 0}, "processing_total": {"avg_ms": 124.325, "count": 186, "max_ms": 159.607, "p95_ms": 151.141}, "read_interval": {"avg_ms": 137.376, "count": 185, "max_ms": 174.203, "p95_ms": 165.101}, "triton_wait": {"avg_ms": 65.418, "count": 186, "max_ms": 98.703, "p95_ms": 91.324}, "visualize": {"avg_ms": 4.811, "count": 186, "max_ms": 6.426, "p95_ms": 5.254}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.947, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3323}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9482.086, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 11, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x555d3c18ff40] left block unavailable for requested intra mode [h264 @ 0x555d3c18ff40] error while decoding MB 0 34, bytestream 4824 [h264 @ 0x555d3c170c00] cabac decode of qscale diff failed at 52 2 [h264 @ 0x555d3c170c00] error while decoding MB 52 2, bytestream 99685 [h264 @ 0x555d3c1b8140] top block unavailable for requested intra mode -1 [h264 @ 0x555d3c1b8140] error while decoding MB 39 0, bytestream 2378 [h264 @ 0x555d3c148880] cabac decode of qscale diff failed at 8 43 [h264 @ 0", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 274, "h264_decode_error_count": 500, "h264_decode_error_count_per_hour": 55, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:49:27+09:00", "last_error_summary": "[h264 @ 0x555d3c18ff40] left block unavailable for requested intra mode [h264 @ 0x555d3c18ff40] error while decoding MB 0 34, bytestream 4824 [h264 @ 0x555d3c170c00] cabac decode of qscale diff failed at 52 2 [h264 @ 0x555d3c170c00] error while decoding MB 52 2, bytestream 99685 [h264 @ 0x555d3c1b8140] top block unavailable for requested intra mode -1 [h264 @ 0x555d3c1b8140] error while decoding MB 39 0, bytestream 2378 [h264 @ 0x555d3c148880] cabac decode of qscale diff failed at 8 43 [h264 @ 0", "last_exit_at": "2026-07-02T07:45:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 238.844, "last_started_at": "2026-07-02T07:45:28+09:00", "pid": 48492, "process": {"exists": false, "pid": 48492}, "restart_count": 306, "restart_count_per_hour": 13, "rtsp_timeout_count": 116, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 307}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.909, "sample_count": 2240, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 398781, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:49:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.068, "last_frame_pushed_at": "2026-07-02T07:49:27+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-02T07:49:27+09:00", "mjpeg": {"age_sec": 0.049, "exists": true, "mtime": "2026-07-02T07:49:27.123897+09:00", "mtime_epoch": 1782946167.123897, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192675}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:49:27+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5547.98, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 86, "pushed_frames": 86, "read_frames": 86, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.875, "pushed": 2.875, "read": 2.875}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.819, "count": 86, "max_ms": 16.483, "p95_ms": 16.246}, "gstreamer_push": {"avg_ms": 1.21, "count": 86, "max_ms": 1.411, "p95_ms": 1.324}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.156, "count": 86, "max_ms": 76.319, "p95_ms": 70.422}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.559, "count": 86, "max_ms": 102.061, "p95_ms": 92.039}, "yolo11_640_fire_11class": {"avg_ms": 66.75, "count": 86, "max_ms": 83.874, "p95_ms": 78.533}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.308, "count": 86, "max_ms": 91.342, "p95_ms": 87.224}}, "truncated_count": 0}, "processing_total": {"avg_ms": 114.845, "count": 86, "max_ms": 160.103, "p95_ms": 148.901}, "read_interval": {"avg_ms": 348.383, "count": 86, "max_ms": 10123.864, "p95_ms": 164.217}, "triton_wait": {"avg_ms": 78.571, "count": 86, "max_ms": 102.073, "p95_ms": 92.051}, "visualize": {"avg_ms": 4.053, "count": 86, "max_ms": 5.22, "p95_ms": 4.99}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.913, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3323}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9481.457, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T07:49:27+09:00", "last_error_summary": "[h264 @ 0x555d3c18ff40] left block unavailable for requested intra mode [h264 @ 0x555d3c18ff40] error while decoding MB 0 34, bytestream 4824 [h264 @ 0x555d3c170c00] cabac decode of qscale diff failed at 52 2 [h264 @ 0x555d3c170c00] error while decoding MB 52 2, bytestream 99685 [h264 @ 0x555d3c1b8140] top block unavailable for requested intra mode -1 [h264 @ 0x555d3c1b8140] error while decoding MB 39 0, bytestream 2378 [h264 @ 0x555d3c148880] cabac decode of qscale diff failed at 8 43 [h264 @ 0", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 274, "h264_decode_error_count": 500, "h264_decode_error_count_per_hour": 55, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:49:27+09:00", "last_error_summary": "[h264 @ 0x555d3c18ff40] left block unavailable for requested intra mode [h264 @ 0x555d3c18ff40] error while decoding MB 0 34, bytestream 4824 [h264 @ 0x555d3c170c00] cabac decode of qscale diff failed at 52 2 [h264 @ 0x555d3c170c00] error while decoding MB 52 2, bytestream 99685 [h264 @ 0x555d3c1b8140] top block unavailable for requested intra mode -1 [h264 @ 0x555d3c1b8140] error while decoding MB 39 0, bytestream 2378 [h264 @ 0x555d3c148880] cabac decode of qscale diff failed at 8 43 [h264 @ 0", "last_exit_at": "2026-07-02T07:49:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 238.905, "last_started_at": "2026-07-02T07:45:28+09:00", "pid": 48492, "process": {"exists": false, "pid": 48492}, "restart_count": 306, "restart_count_per_hour": 13, "rtsp_timeout_count": 116, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 307}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.909, "sample_count": 2240, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 398782, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:49:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T07:49:27+09:00", "last_frame_read_age_sec": 0.115, "last_frame_read_at": "2026-07-02T07:49:27+09:00", "mjpeg": {"age_sec": 0.111, "exists": true, "mtime": "2026-07-02T07:49:27.123897+09:00", "mtime_epoch": 1782946167.123897, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192675}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:49:27+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5547.98, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 87, "pushed_frames": 87, "read_frames": 86, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.902, "pushed": 2.902, "read": 2.869}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.809, "count": 87, "max_ms": 16.483, "p95_ms": 16.246}, "gstreamer_push": {"avg_ms": 1.211, "count": 87, "max_ms": 1.411, "p95_ms": 1.33}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.248, "count": 87, "max_ms": 76.319, "p95_ms": 70.422}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.635, "count": 87, "max_ms": 102.061, "p95_ms": 92.039}, "yolo11_640_fire_11class": {"avg_ms": 66.862, "count": 87, "max_ms": 83.874, "p95_ms": 78.533}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.392, "count": 87, "max_ms": 91.342, "p95_ms": 87.224}}, "truncated_count": 0}, "processing_total": {"avg_ms": 114.763, "count": 87, "max_ms": 160.103, "p95_ms": 148.901}, "read_interval": {"avg_ms": 348.383, "count": 86, "max_ms": 10123.864, "p95_ms": 164.217}, "triton_wait": {"avg_ms": 78.647, "count": 87, "max_ms": 102.073, "p95_ms": 92.051}, "visualize": {"avg_ms": 4.048, "count": 87, "max_ms": 5.22, "p95_ms": 4.99}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.974, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3323}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9481.383, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 274, "h264_decode_error_count": 500, "h264_decode_error_count_per_hour": 55, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:49:27+09:00", "last_error_summary": "[h264 @ 0x555d3c18ff40] left block unavailable for requested intra mode [h264 @ 0x555d3c18ff40] error while decoding MB 0 34, bytestream 4824 [h264 @ 0x555d3c170c00] cabac decode of qscale diff failed at 52 2 [h264 @ 0x555d3c170c00] error while decoding MB 52 2, bytestream 99685 [h264 @ 0x555d3c1b8140] top block unavailable for requested intra mode -1 [h264 @ 0x555d3c1b8140] error while decoding MB 39 0, bytestream 2378 [h264 @ 0x555d3c148880] cabac decode of qscale diff failed at 8 43 [h264 @ 0", "last_exit_at": "2026-07-02T07:49:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 238.913, "last_started_at": "2026-07-02T07:45:28+09:00", "pid": 48492, "process": {"exists": false, "pid": 48492}, "restart_count": 306, "restart_count_per_hour": 13, "rtsp_timeout_count": 116, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 307}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.909, "sample_count": 2240, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 398782, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:49:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-02T07:49:27+09:00", "last_frame_read_age_sec": 0.123, "last_frame_read_at": "2026-07-02T07:49:27+09:00", "mjpeg": {"age_sec": 0.118, "exists": true, "mtime": "2026-07-02T07:49:27.123897+09:00", "mtime_epoch": 1782946167.123897, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192675}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:49:27+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5547.98, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 87, "pushed_frames": 87, "read_frames": 86, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.902, "pushed": 2.902, "read": 2.868}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.809, "count": 87, "max_ms": 16.483, "p95_ms": 16.246}, "gstreamer_push": {"avg_ms": 1.211, "count": 87, "max_ms": 1.411, "p95_ms": 1.33}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.248, "count": 87, "max_ms": 76.319, "p95_ms": 70.422}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.635, "count": 87, "max_ms": 102.061, "p95_ms": 92.039}, "yolo11_640_fire_11class": {"avg_ms": 66.862, "count": 87, "max_ms": 83.874, "p95_ms": 78.533}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.392, "count": 87, "max_ms": 91.342, "p95_ms": 87.224}}, "truncated_count": 0}, "processing_total": {"avg_ms": 114.763, "count": 87, "max_ms": 160.103, "p95_ms": 148.901}, "read_interval": {"avg_ms": 348.383, "count": 86, "max_ms": 10123.864, "p95_ms": 164.217}, "triton_wait": {"avg_ms": 78.647, "count": 87, "max_ms": 102.073, "p95_ms": 92.051}, "visualize": {"avg_ms": 4.048, "count": 87, "max_ms": 5.22, "p95_ms": 4.99}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.982, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3323}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9586.324, "limit_mb": 8796093022207.996}, "detail": {"pid": 48599, "restart_count": 307, "start_count": 308}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 274, "h264_decode_error_count": 500, "h264_decode_error_count_per_hour": 55, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:49:27+09:00", "last_error_summary": "[h264 @ 0x555d3c18ff40] left block unavailable for requested intra mode [h264 @ 0x555d3c18ff40] error while decoding MB 0 34, bytestream 4824 [h264 @ 0x555d3c170c00] cabac decode of qscale diff failed at 52 2 [h264 @ 0x555d3c170c00] error while decoding MB 52 2, bytestream 99685 [h264 @ 0x555d3c1b8140] top block unavailable for requested intra mode -1 [h264 @ 0x555d3c1b8140] error while decoding MB 39 0, bytestream 2378 [h264 @ 0x555d3c148880] cabac decode of qscale diff failed at 8 43 [h264 @ 0", "last_exit_at": "2026-07-02T07:49:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T07:49:29+09:00", "pid": 48599, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.723, "name": "ffmpeg", "pid": 48599, "rss_mb": 138.723, "state": "S (sleeping)", "threads": 106}, "restart_count": 307, "restart_count_per_hour": 14, "rtsp_timeout_count": 116, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 308}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.187, "sample_count": 2199, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 398782, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:49:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.387, "last_frame_pushed_at": "2026-07-02T07:49:27+09:00", "last_frame_read_age_sec": 4.497, "last_frame_read_at": "2026-07-02T07:49:27+09:00", "mjpeg": {"age_sec": 4.369, "exists": true, "mtime": "2026-07-02T07:49:27.247899+09:00", "mtime_epoch": 1782946167.2478993, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193040}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:49:31+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5547.98, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 56, "pushed_frames": 56, "read_frames": 55, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.872, "pushed": 1.872, "read": 1.838}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.426, "count": 56, "max_ms": 16.246, "p95_ms": 16.202}, "gstreamer_push": {"avg_ms": 1.247, "count": 56, "max_ms": 1.411, "p95_ms": 1.351}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 57.67, "count": 56, "max_ms": 75.985, "p95_ms": 73.044}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.456, "count": 56, "max_ms": 93.025, "p95_ms": 92.039}, "yolo11_640_fire_11class": {"avg_ms": 67.774, "count": 56, "max_ms": 82.294, "p95_ms": 78.667}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.456, "count": 56, "max_ms": 88.011, "p95_ms": 87.224}}, "truncated_count": 0}, "processing_total": {"avg_ms": 107.381, "count": 56, "max_ms": 151.356, "p95_ms": 145.327}, "read_interval": {"avg_ms": 464.829, "count": 55, "max_ms": 10123.864, "p95_ms": 164.217}, "triton_wait": {"avg_ms": 79.469, "count": 56, "max_ms": 93.038, "p95_ms": 92.051}, "visualize": {"avg_ms": 3.73, "count": 56, "max_ms": 4.918, "p95_ms": 4.892}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.918, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3335}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9483.855, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x56500d8a78c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56500d8a78c0] error while decoding MB 0 39, bytestream 1963 [h264 @ 0x56500cf8b940] left block unavailable for requested intra mode [h264 @ 0x56500cf8b940] error while decoding MB 0 13, bytestream 1904 [in#0/rtsp @ 0x56500ce26e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56500ce26e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 275, "h264_decode_error_count": 502, "h264_decode_error_count_per_hour": 56, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:52:49+09:00", "last_error_summary": "[h264 @ 0x56500d8a78c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56500d8a78c0] error while decoding MB 0 39, bytestream 1963 [h264 @ 0x56500cf8b940] left block unavailable for requested intra mode [h264 @ 0x56500cf8b940] error while decoding MB 0 13, bytestream 1904 [in#0/rtsp @ 0x56500ce26e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56500ce26e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T07:49:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 199.549, "last_started_at": "2026-07-02T07:49:29+09:00", "pid": 48599, "process": {"exists": false, "pid": 48599}, "restart_count": 307, "restart_count_per_hour": 13, "rtsp_timeout_count": 118, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 308}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.514, "sample_count": 2153, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 400275, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:52:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.074, "last_frame_pushed_at": "2026-07-02T07:52:49+09:00", "last_frame_read_age_sec": 0.063, "last_frame_read_at": "2026-07-02T07:52:49+09:00", "mjpeg": {"age_sec": 0.057, "exists": true, "mtime": "2026-07-02T07:52:49.107760+09:00", "mtime_epoch": 1782946369.10776, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193467}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:52:49+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5549.012, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 92, "pushed_frames": 92, "read_frames": 93, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.067, "pushed": 3.067, "read": 3.101}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.21, "count": 92, "max_ms": 19.007, "p95_ms": 17.022}, "gstreamer_push": {"avg_ms": 1.261, "count": 92, "max_ms": 1.353, "p95_ms": 1.345}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.705, "count": 92, "max_ms": 72.594, "p95_ms": 69.52}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.743, "count": 92, "max_ms": 91.086, "p95_ms": 88.979}, "yolo11_640_fire_11class": {"avg_ms": 66.506, "count": 92, "max_ms": 75.864, "p95_ms": 72.179}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.605, "count": 92, "max_ms": 88.9, "p95_ms": 80.86}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.412, "count": 92, "max_ms": 116.878, "p95_ms": 111.212}, "read_interval": {"avg_ms": 323.119, "count": 93, "max_ms": 10211.271, "p95_ms": 127.373}, "triton_wait": {"avg_ms": 81.756, "count": 92, "max_ms": 91.098, "p95_ms": 88.991}, "visualize": {"avg_ms": 3.578, "count": 92, "max_ms": 4.471, "p95_ms": 4.13}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3335}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9483.066, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T07:52:49+09:00", "last_error_summary": "[h264 @ 0x56500d8a78c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56500d8a78c0] error while decoding MB 0 39, bytestream 1963 [h264 @ 0x56500cf8b940] left block unavailable for requested intra mode [h264 @ 0x56500cf8b940] error while decoding MB 0 13, bytestream 1904 [in#0/rtsp @ 0x56500ce26e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56500ce26e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 275, "h264_decode_error_count": 502, "h264_decode_error_count_per_hour": 56, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:52:49+09:00", "last_error_summary": "[h264 @ 0x56500d8a78c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56500d8a78c0] error while decoding MB 0 39, bytestream 1963 [h264 @ 0x56500cf8b940] left block unavailable for requested intra mode [h264 @ 0x56500cf8b940] error while decoding MB 0 13, bytestream 1904 [in#0/rtsp @ 0x56500ce26e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56500ce26e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T07:52:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 199.606, "last_started_at": "2026-07-02T07:49:29+09:00", "pid": 48599, "process": {"exists": false, "pid": 48599}, "restart_count": 307, "restart_count_per_hour": 13, "rtsp_timeout_count": 118, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 308}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.514, "sample_count": 2154, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 400276, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:52:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T07:52:49+09:00", "last_frame_read_age_sec": 0.119, "last_frame_read_at": "2026-07-02T07:52:49+09:00", "mjpeg": {"age_sec": 0.113, "exists": true, "mtime": "2026-07-02T07:52:49.107760+09:00", "mtime_epoch": 1782946369.10776, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193467}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:52:49+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5549.012, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 93, "pushed_frames": 93, "read_frames": 92, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.106, "pushed": 3.106, "read": 3.072}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.207, "count": 93, "max_ms": 19.007, "p95_ms": 17.022}, "gstreamer_push": {"avg_ms": 1.261, "count": 93, "max_ms": 1.353, "p95_ms": 1.345}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.774, "count": 93, "max_ms": 72.594, "p95_ms": 69.52}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.823, "count": 93, "max_ms": 91.086, "p95_ms": 89.126}, "yolo11_640_fire_11class": {"avg_ms": 66.56, "count": 93, "max_ms": 75.864, "p95_ms": 72.179}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.693, "count": 93, "max_ms": 88.9, "p95_ms": 80.86}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.489, "count": 93, "max_ms": 116.878, "p95_ms": 111.477}, "read_interval": {"avg_ms": 325.34, "count": 92, "max_ms": 10211.271, "p95_ms": 127.373}, "triton_wait": {"avg_ms": 81.835, "count": 93, "max_ms": 91.098, "p95_ms": 89.137}, "visualize": {"avg_ms": 3.579, "count": 93, "max_ms": 4.471, "p95_ms": 4.13}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.946, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3335}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9483.105, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 275, "h264_decode_error_count": 502, "h264_decode_error_count_per_hour": 56, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:52:49+09:00", "last_error_summary": "[h264 @ 0x56500d8a78c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56500d8a78c0] error while decoding MB 0 39, bytestream 1963 [h264 @ 0x56500cf8b940] left block unavailable for requested intra mode [h264 @ 0x56500cf8b940] error while decoding MB 0 13, bytestream 1904 [in#0/rtsp @ 0x56500ce26e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56500ce26e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T07:52:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 199.613, "last_started_at": "2026-07-02T07:49:29+09:00", "pid": 48599, "process": {"exists": false, "pid": 48599}, "restart_count": 307, "restart_count_per_hour": 13, "rtsp_timeout_count": 118, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 308}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.514, "sample_count": 2154, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 400276, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:52:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-02T07:52:49+09:00", "last_frame_read_age_sec": 0.127, "last_frame_read_at": "2026-07-02T07:52:49+09:00", "mjpeg": {"age_sec": 0.121, "exists": true, "mtime": "2026-07-02T07:52:49.107760+09:00", "mtime_epoch": 1782946369.10776, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193467}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:52:49+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5549.012, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 93, "pushed_frames": 93, "read_frames": 92, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.105, "pushed": 3.105, "read": 3.071}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.207, "count": 93, "max_ms": 19.007, "p95_ms": 17.022}, "gstreamer_push": {"avg_ms": 1.261, "count": 93, "max_ms": 1.353, "p95_ms": 1.345}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.774, "count": 93, "max_ms": 72.594, "p95_ms": 69.52}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.823, "count": 93, "max_ms": 91.086, "p95_ms": 89.126}, "yolo11_640_fire_11class": {"avg_ms": 66.56, "count": 93, "max_ms": 75.864, "p95_ms": 72.179}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.693, "count": 93, "max_ms": 88.9, "p95_ms": 80.86}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.489, "count": 93, "max_ms": 116.878, "p95_ms": 111.477}, "read_interval": {"avg_ms": 325.34, "count": 92, "max_ms": 10211.271, "p95_ms": 127.373}, "triton_wait": {"avg_ms": 81.835, "count": 93, "max_ms": 91.098, "p95_ms": 89.137}, "visualize": {"avg_ms": 3.579, "count": 93, "max_ms": 4.471, "p95_ms": 4.13}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.954, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3335}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9589.168, "limit_mb": 8796093022207.996}, "detail": {"pid": 48706, "restart_count": 308, "start_count": 309}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 275, "h264_decode_error_count": 502, "h264_decode_error_count_per_hour": 56, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:52:49+09:00", "last_error_summary": "[h264 @ 0x56500d8a78c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x56500d8a78c0] error while decoding MB 0 39, bytestream 1963 [h264 @ 0x56500cf8b940] left block unavailable for requested intra mode [h264 @ 0x56500cf8b940] error while decoding MB 0 13, bytestream 1904 [in#0/rtsp @ 0x56500ce26e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56500ce26e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T07:52:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T07:52:51+09:00", "pid": 48706, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.383, "name": "ffmpeg", "pid": 48706, "rss_mb": 136.383, "state": "S (sleeping)", "threads": 106}, "restart_count": 308, "restart_count_per_hour": 14, "rtsp_timeout_count": 118, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 309}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.516, "sample_count": 2115, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 400276, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:52:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.578, "last_frame_pushed_at": "2026-07-02T07:52:49+09:00", "last_frame_read_age_sec": 4.692, "last_frame_read_at": "2026-07-02T07:52:49+09:00", "mjpeg": {"age_sec": 4.558, "exists": true, "mtime": "2026-07-02T07:52:49.235762+09:00", "mtime_epoch": 1782946369.2357624, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195566}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T07:52:53+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5549.012, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 54, "pushed_frames": 54, "read_frames": 54, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.801, "pushed": 1.801, "read": 1.801}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.437, "count": 54, "max_ms": 19.007, "p95_ms": 18.566}, "gstreamer_push": {"avg_ms": 1.266, "count": 54, "max_ms": 1.348, "p95_ms": 1.345}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.372, "count": 54, "max_ms": 70.536, "p95_ms": 69.98}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.555, "count": 54, "max_ms": 91.086, "p95_ms": 89.767}, "yolo11_640_fire_11class": {"avg_ms": 66.131, "count": 54, "max_ms": 75.864, "p95_ms": 72.091}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.231, "count": 54, "max_ms": 88.9, "p95_ms": 81.358}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.459, "count": 54, "max_ms": 116.878, "p95_ms": 113.061}, "read_interval": {"avg_ms": 470.738, "count": 54, "max_ms": 10211.271, "p95_ms": 132.265}, "triton_wait": {"avg_ms": 82.567, "count": 54, "max_ms": 91.098, "p95_ms": 89.781}, "visualize": {"avg_ms": 3.593, "count": 54, "max_ms": 4.316, "p95_ms": 4.13}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.99, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 9, "tracking_summary": 4764}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9442.18, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x56180164f000] error while decoding MB 62 67, bytestream -21 [in#0/rtsp @ 0x5618015d4e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5618015d4e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 75, "h264_decode_error_count": 75, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:57:36+09:00", "last_error_summary": "[h264 @ 0x56180164f000] error while decoding MB 62 67, bytestream -21 [in#0/rtsp @ 0x5618015d4e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5618015d4e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T07:33:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 1417.802, "last_started_at": "2026-07-02T07:33:58+09:00", "pid": 48233, "process": {"exists": false, "pid": 48233}, "restart_count": 74, "restart_count_per_hour": 7, "rtsp_timeout_count": 136, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 75}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.188, "sample_count": 2383, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 571777, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:57:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-02T07:57:36+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-02T07:57:36+09:00", "mjpeg": {"age_sec": 0.059, "exists": true, "mtime": "2026-07-02T07:57:36.561237+09:00", "mtime_epoch": 1782946656.5612366, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 220403}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T07:57:36+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1384.668, "name": "celery", "pid": 267, "rss_mb": 1142.145, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 240, "pushed_frames": 240, "read_frames": 240, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.013, "pushed": 8.013, "read": 8.013}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.386, "count": 240, "max_ms": 6.926, "p95_ms": 5.579}, "gstreamer_push": {"avg_ms": 1.602, "count": 240, "max_ms": 6.369, "p95_ms": 3.691}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.638, "count": 240, "max_ms": 75.414, "p95_ms": 72.18}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 94.785, "count": 240, "max_ms": 101.185, "p95_ms": 97.965}, "yolo11_640_fire_11class": {"avg_ms": 74.127, "count": 240, "max_ms": 83.651, "p95_ms": 79.915}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.876, "count": 240, "max_ms": 88.487, "p95_ms": 85.138}}, "truncated_count": 0}, "processing_total": {"avg_ms": 108.938, "count": 240, "max_ms": 119.146, "p95_ms": 112.221}, "read_interval": {"avg_ms": 124.994, "count": 240, "max_ms": 139.63, "p95_ms": 129.631}, "triton_wait": {"avg_ms": 94.798, "count": 240, "max_ms": 101.197, "p95_ms": 97.978}, "visualize": {"avg_ms": 3.685, "count": 240, "max_ms": 4.606, "p95_ms": 3.906}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.952, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 9, "tracking_summary": 4764}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9441.586, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T07:57:36+09:00", "last_error_summary": "[h264 @ 0x56180164f000] error while decoding MB 62 67, bytestream -21 [in#0/rtsp @ 0x5618015d4e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5618015d4e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 75, "h264_decode_error_count": 75, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:57:36+09:00", "last_error_summary": "[h264 @ 0x56180164f000] error while decoding MB 62 67, bytestream -21 [in#0/rtsp @ 0x5618015d4e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5618015d4e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T07:57:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 1417.856, "last_started_at": "2026-07-02T07:33:58+09:00", "pid": 48233, "process": {"exists": false, "pid": 48233}, "restart_count": 74, "restart_count_per_hour": 7, "rtsp_timeout_count": 136, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 75}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.188, "sample_count": 2383, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 571778, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:57:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T07:57:36+09:00", "last_frame_read_age_sec": 0.118, "last_frame_read_at": "2026-07-02T07:57:36+09:00", "mjpeg": {"age_sec": 0.114, "exists": true, "mtime": "2026-07-02T07:57:36.561237+09:00", "mtime_epoch": 1782946656.5612366, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 220403}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T07:57:36+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1384.668, "name": "celery", "pid": 267, "rss_mb": 1142.145, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 240, "pushed_frames": 240, "read_frames": 240, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.002, "pushed": 8.002, "read": 8.002}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.386, "count": 240, "max_ms": 6.926, "p95_ms": 5.579}, "gstreamer_push": {"avg_ms": 1.602, "count": 240, "max_ms": 6.369, "p95_ms": 3.691}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.655, "count": 240, "max_ms": 75.414, "p95_ms": 72.18}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 94.781, "count": 240, "max_ms": 101.185, "p95_ms": 97.965}, "yolo11_640_fire_11class": {"avg_ms": 74.115, "count": 240, "max_ms": 83.651, "p95_ms": 79.915}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.824, "count": 240, "max_ms": 88.487, "p95_ms": 85.138}}, "truncated_count": 0}, "processing_total": {"avg_ms": 108.933, "count": 240, "max_ms": 119.146, "p95_ms": 112.221}, "read_interval": {"avg_ms": 124.994, "count": 240, "max_ms": 139.63, "p95_ms": 129.631}, "triton_wait": {"avg_ms": 94.794, "count": 240, "max_ms": 101.197, "p95_ms": 97.978}, "visualize": {"avg_ms": 3.685, "count": 240, "max_ms": 4.606, "p95_ms": 3.906}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 9, "tracking_summary": 4764}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9441.551, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 75, "h264_decode_error_count": 75, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:57:36+09:00", "last_error_summary": "[h264 @ 0x56180164f000] error while decoding MB 62 67, bytestream -21 [in#0/rtsp @ 0x5618015d4e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5618015d4e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T07:57:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 1417.864, "last_started_at": "2026-07-02T07:33:58+09:00", "pid": 48233, "process": {"exists": false, "pid": 48233}, "restart_count": 74, "restart_count_per_hour": 7, "rtsp_timeout_count": 136, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 75}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.188, "sample_count": 2383, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 571778, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:57:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-02T07:57:36+09:00", "last_frame_read_age_sec": 0.127, "last_frame_read_at": "2026-07-02T07:57:36+09:00", "mjpeg": {"age_sec": 0.122, "exists": true, "mtime": "2026-07-02T07:57:36.561237+09:00", "mtime_epoch": 1782946656.5612366, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 220403}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T07:57:36+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1384.668, "name": "celery", "pid": 267, "rss_mb": 1142.145, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 240, "pushed_frames": 240, "read_frames": 240, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.0, "pushed": 8.0, "read": 8.0}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.386, "count": 240, "max_ms": 6.926, "p95_ms": 5.579}, "gstreamer_push": {"avg_ms": 1.602, "count": 240, "max_ms": 6.369, "p95_ms": 3.691}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.655, "count": 240, "max_ms": 75.414, "p95_ms": 72.18}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 94.781, "count": 240, "max_ms": 101.185, "p95_ms": 97.965}, "yolo11_640_fire_11class": {"avg_ms": 74.115, "count": 240, "max_ms": 83.651, "p95_ms": 79.915}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.824, "count": 240, "max_ms": 88.487, "p95_ms": 85.138}}, "truncated_count": 0}, "processing_total": {"avg_ms": 108.933, "count": 240, "max_ms": 119.146, "p95_ms": 112.221}, "read_interval": {"avg_ms": 124.994, "count": 240, "max_ms": 139.63, "p95_ms": 129.631}, "triton_wait": {"avg_ms": 94.794, "count": 240, "max_ms": 101.197, "p95_ms": 97.978}, "visualize": {"avg_ms": 3.685, "count": 240, "max_ms": 4.606, "p95_ms": 3.906}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 9, "tracking_summary": 4764}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9564.855, "limit_mb": 8796093022207.996}, "detail": {"pid": 48813, "restart_count": 75, "start_count": 76}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 75, "h264_decode_error_count": 75, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T07:57:36+09:00", "last_error_summary": "[h264 @ 0x56180164f000] error while decoding MB 62 67, bytestream -21 [in#0/rtsp @ 0x5618015d4e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5618015d4e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T07:57:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T07:57:39+09:00", "pid": 48813, "process": {"exists": true, "fd_count": 5, "hwm_mb": 156.73, "name": "ffmpeg", "pid": 48813, "rss_mb": 156.578, "state": "R (running)", "threads": 49}, "restart_count": 75, "restart_count_per_hour": 8, "rtsp_timeout_count": 136, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 76}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.226, "sample_count": 2328, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 571778, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T07:57:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.492, "last_frame_pushed_at": "2026-07-02T07:57:36+09:00", "last_frame_read_age_sec": 4.603, "last_frame_read_at": "2026-07-02T07:57:36+09:00", "mjpeg": {"age_sec": 4.474, "exists": true, "mtime": "2026-07-02T07:57:36.685239+09:00", "mtime_epoch": 1782946656.685239, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 220403}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T07:57:41+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1384.668, "name": "celery", "pid": 267, "rss_mb": 1142.145, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 205, "pushed_frames": 205, "read_frames": 204, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.849, "pushed": 6.849, "read": 6.815}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.386, "count": 205, "max_ms": 6.926, "p95_ms": 5.544}, "gstreamer_push": {"avg_ms": 1.567, "count": 205, "max_ms": 6.369, "p95_ms": 3.645}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.491, "count": 205, "max_ms": 75.414, "p95_ms": 72.18}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 94.576, "count": 205, "max_ms": 99.901, "p95_ms": 97.638}, "yolo11_640_fire_11class": {"avg_ms": 73.48, "count": 205, "max_ms": 83.651, "p95_ms": 79.833}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.646, "count": 205, "max_ms": 88.487, "p95_ms": 85.484}}, "truncated_count": 0}, "processing_total": {"avg_ms": 108.711, "count": 205, "max_ms": 113.965, "p95_ms": 111.992}, "read_interval": {"avg_ms": 124.723, "count": 204, "max_ms": 132.261, "p95_ms": 129.279}, "triton_wait": {"avg_ms": 94.588, "count": 205, "max_ms": 99.916, "p95_ms": 97.651}, "visualize": {"avg_ms": 3.683, "count": 205, "max_ms": 4.482, "p95_ms": 3.938}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.933, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3365}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9494.102, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 7, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55af3f337d00] cabac decode of qscale diff failed at 73 43 [h264 @ 0x55af3f337d00] error while decoding MB 73 43, bytestream 2786 [h264 @ 0x55af3f34bec0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55af3f34bec0] error while decoding MB 0 4, bytestream 41592 [h264 @ 0x55af3f323cc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55af3f323cc0] error while decoding MB 0 17, bytestream 4133 [h264 @ 0x55af3f2723c0] top block unavailable for requested intra", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 276, "h264_decode_error_count": 509, "h264_decode_error_count_per_hour": 62, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:00:05+09:00", "last_error_summary": "[h264 @ 0x55af3f337d00] cabac decode of qscale diff failed at 73 43 [h264 @ 0x55af3f337d00] error while decoding MB 73 43, bytestream 2786 [h264 @ 0x55af3f34bec0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55af3f34bec0] error while decoding MB 0 4, bytestream 41592 [h264 @ 0x55af3f323cc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55af3f323cc0] error while decoding MB 0 17, bytestream 4133 [h264 @ 0x55af3f2723c0] top block unavailable for requested intra", "last_exit_at": "2026-07-02T07:52:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 433.987, "last_started_at": "2026-07-02T07:52:51+09:00", "pid": 48706, "process": {"exists": false, "pid": 48706}, "restart_count": 308, "restart_count_per_hour": 12, "rtsp_timeout_count": 120, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 309}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.731, "sample_count": 2465, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 403840, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:00:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.092, "last_frame_pushed_at": "2026-07-02T08:00:05+09:00", "last_frame_read_age_sec": 0.08, "last_frame_read_at": "2026-07-02T08:00:05+09:00", "mjpeg": {"age_sec": 0.072, "exists": true, "mtime": "2026-07-02T08:00:05.707245+09:00", "mtime_epoch": 1782946805.7072446, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 215436}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:00:05+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5550.559, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 85, "pushed_frames": 85, "read_frames": 86, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.834, "pushed": 2.834, "read": 2.868}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.036, "count": 85, "max_ms": 17.638, "p95_ms": 15.268}, "gstreamer_push": {"avg_ms": 1.258, "count": 85, "max_ms": 1.388, "p95_ms": 1.321}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.584, "count": 85, "max_ms": 70.949, "p95_ms": 66.694}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.994, "count": 85, "max_ms": 95.134, "p95_ms": 89.186}, "yolo11_640_fire_11class": {"avg_ms": 68.039, "count": 85, "max_ms": 78.496, "p95_ms": 75.519}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.893, "count": 85, "max_ms": 84.413, "p95_ms": 80.872}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.464, "count": 85, "max_ms": 116.993, "p95_ms": 111.832}, "read_interval": {"avg_ms": 348.933, "count": 86, "max_ms": 19986.721, "p95_ms": 127.14}, "triton_wait": {"avg_ms": 81.006, "count": 85, "max_ms": 95.146, "p95_ms": 89.198}, "visualize": {"avg_ms": 3.557, "count": 85, "max_ms": 4.313, "p95_ms": 3.776}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.99, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3365}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9493.801, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T08:00:05+09:00", "last_error_summary": "[h264 @ 0x55af3f337d00] cabac decode of qscale diff failed at 73 43 [h264 @ 0x55af3f337d00] error while decoding MB 73 43, bytestream 2786 [h264 @ 0x55af3f34bec0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55af3f34bec0] error while decoding MB 0 4, bytestream 41592 [h264 @ 0x55af3f323cc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55af3f323cc0] error while decoding MB 0 17, bytestream 4133 [h264 @ 0x55af3f2723c0] top block unavailable for requested intra", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 276, "h264_decode_error_count": 509, "h264_decode_error_count_per_hour": 62, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:00:05+09:00", "last_error_summary": "[h264 @ 0x55af3f337d00] cabac decode of qscale diff failed at 73 43 [h264 @ 0x55af3f337d00] error while decoding MB 73 43, bytestream 2786 [h264 @ 0x55af3f34bec0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55af3f34bec0] error while decoding MB 0 4, bytestream 41592 [h264 @ 0x55af3f323cc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55af3f323cc0] error while decoding MB 0 17, bytestream 4133 [h264 @ 0x55af3f2723c0] top block unavailable for requested intra", "last_exit_at": "2026-07-02T08:00:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 434.027, "last_started_at": "2026-07-02T07:52:51+09:00", "pid": 48706, "process": {"exists": false, "pid": 48706}, "restart_count": 308, "restart_count_per_hour": 12, "rtsp_timeout_count": 120, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 309}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.731, "sample_count": 2465, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 403841, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:00:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T08:00:05+09:00", "last_frame_read_age_sec": 0.12, "last_frame_read_at": "2026-07-02T08:00:05+09:00", "mjpeg": {"age_sec": 0.112, "exists": true, "mtime": "2026-07-02T08:00:05.707245+09:00", "mtime_epoch": 1782946805.7072446, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 215436}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:00:05+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5550.559, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 86, "pushed_frames": 86, "read_frames": 85, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.872, "pushed": 2.872, "read": 2.839}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.032, "count": 86, "max_ms": 17.638, "p95_ms": 15.268}, "gstreamer_push": {"avg_ms": 1.259, "count": 86, "max_ms": 1.388, "p95_ms": 1.321}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.653, "count": 86, "max_ms": 70.949, "p95_ms": 66.694}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.103, "count": 86, "max_ms": 95.134, "p95_ms": 89.389}, "yolo11_640_fire_11class": {"avg_ms": 68.003, "count": 86, "max_ms": 78.496, "p95_ms": 75.519}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.931, "count": 86, "max_ms": 84.413, "p95_ms": 80.872}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.567, "count": 86, "max_ms": 116.993, "p95_ms": 112.36}, "read_interval": {"avg_ms": 351.873, "count": 85, "max_ms": 19986.721, "p95_ms": 127.14}, "triton_wait": {"avg_ms": 81.115, "count": 86, "max_ms": 95.146, "p95_ms": 89.4}, "visualize": {"avg_ms": 3.556, "count": 86, "max_ms": 4.313, "p95_ms": 3.776}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.944, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3365}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9493.703, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 276, "h264_decode_error_count": 509, "h264_decode_error_count_per_hour": 62, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:00:05+09:00", "last_error_summary": "[h264 @ 0x55af3f337d00] cabac decode of qscale diff failed at 73 43 [h264 @ 0x55af3f337d00] error while decoding MB 73 43, bytestream 2786 [h264 @ 0x55af3f34bec0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55af3f34bec0] error while decoding MB 0 4, bytestream 41592 [h264 @ 0x55af3f323cc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55af3f323cc0] error while decoding MB 0 17, bytestream 4133 [h264 @ 0x55af3f2723c0] top block unavailable for requested intra", "last_exit_at": "2026-07-02T08:00:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 434.035, "last_started_at": "2026-07-02T07:52:51+09:00", "pid": 48706, "process": {"exists": false, "pid": 48706}, "restart_count": 308, "restart_count_per_hour": 12, "rtsp_timeout_count": 120, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 309}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.731, "sample_count": 2465, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 403841, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:00:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-02T08:00:05+09:00", "last_frame_read_age_sec": 0.128, "last_frame_read_at": "2026-07-02T08:00:05+09:00", "mjpeg": {"age_sec": 0.12, "exists": true, "mtime": "2026-07-02T08:00:05.707245+09:00", "mtime_epoch": 1782946805.7072446, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 215436}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:00:05+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5550.559, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 86, "pushed_frames": 86, "read_frames": 85, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.871, "pushed": 2.871, "read": 2.838}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.032, "count": 86, "max_ms": 17.638, "p95_ms": 15.268}, "gstreamer_push": {"avg_ms": 1.259, "count": 86, "max_ms": 1.388, "p95_ms": 1.321}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.653, "count": 86, "max_ms": 70.949, "p95_ms": 66.694}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.103, "count": 86, "max_ms": 95.134, "p95_ms": 89.389}, "yolo11_640_fire_11class": {"avg_ms": 68.003, "count": 86, "max_ms": 78.496, "p95_ms": 75.519}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.931, "count": 86, "max_ms": 84.413, "p95_ms": 80.872}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.567, "count": 86, "max_ms": 116.993, "p95_ms": 112.36}, "read_interval": {"avg_ms": 351.873, "count": 85, "max_ms": 19986.721, "p95_ms": 127.14}, "triton_wait": {"avg_ms": 81.115, "count": 86, "max_ms": 95.146, "p95_ms": 89.4}, "visualize": {"avg_ms": 3.556, "count": 86, "max_ms": 4.313, "p95_ms": 3.776}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.952, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3365}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9593.848, "limit_mb": 8796093022207.996}, "detail": {"pid": 49025, "restart_count": 309, "start_count": 310}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 276, "h264_decode_error_count": 509, "h264_decode_error_count_per_hour": 62, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:00:05+09:00", "last_error_summary": "[h264 @ 0x55af3f337d00] cabac decode of qscale diff failed at 73 43 [h264 @ 0x55af3f337d00] error while decoding MB 73 43, bytestream 2786 [h264 @ 0x55af3f34bec0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55af3f34bec0] error while decoding MB 0 4, bytestream 41592 [h264 @ 0x55af3f323cc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55af3f323cc0] error while decoding MB 0 17, bytestream 4133 [h264 @ 0x55af3f2723c0] top block unavailable for requested intra", "last_exit_at": "2026-07-02T08:00:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T08:00:08+09:00", "pid": 49025, "process": {"exists": true, "fd_count": 5, "hwm_mb": 134.086, "name": "ffmpeg", "pid": 49025, "rss_mb": 134.086, "state": "S (sleeping)", "threads": 106}, "restart_count": 309, "restart_count_per_hour": 13, "rtsp_timeout_count": 120, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 310}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 114.744, "sample_count": 2426, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 403841, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:00:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.758, "last_frame_pushed_at": "2026-07-02T08:00:05+09:00", "last_frame_read_age_sec": 4.872, "last_frame_read_at": "2026-07-02T08:00:05+09:00", "mjpeg": {"age_sec": 4.74, "exists": true, "mtime": "2026-07-02T08:00:05.831244+09:00", "mtime_epoch": 1782946805.8312445, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 215424}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:00:10+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5550.559, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 68, "pushed_frames": 68, "read_frames": 68, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.267, "pushed": 5.267, "read": 5.267}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.073, "count": 68, "max_ms": 15.382, "p95_ms": 15.16}, "gstreamer_push": {"avg_ms": 1.258, "count": 68, "max_ms": 1.337, "p95_ms": 1.321}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.709, "count": 68, "max_ms": 70.949, "p95_ms": 66.694}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 83.421, "count": 68, "max_ms": 95.134, "p95_ms": 89.389}, "yolo11_640_fire_11class": {"avg_ms": 70.011, "count": 68, "max_ms": 78.496, "p95_ms": 75.89}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.827, "count": 68, "max_ms": 83.877, "p95_ms": 80.815}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.949, "count": 68, "max_ms": 116.993, "p95_ms": 112.36}, "read_interval": {"avg_ms": 412.134, "count": 68, "max_ms": 19986.721, "p95_ms": 127.14}, "triton_wait": {"avg_ms": 83.434, "count": 68, "max_ms": 95.146, "p95_ms": 89.4}, "visualize": {"avg_ms": 3.556, "count": 68, "max_ms": 4.086, "p95_ms": 3.776}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 12.911, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3384}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9491.914, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55ace968be00] error while decoding MB 3 39, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 277, "h264_decode_error_count": 510, "h264_decode_error_count_per_hour": 62, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:04:45+09:00", "last_error_summary": "[h264 @ 0x55ace968be00] error while decoding MB 3 39, bytestream -7", "last_exit_at": "2026-07-02T08:00:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 277.106, "last_started_at": "2026-07-02T08:00:08+09:00", "pid": 49025, "process": {"exists": false, "pid": 49025}, "restart_count": 309, "restart_count_per_hour": 12, "rtsp_timeout_count": 120, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 310}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 128.912, "sample_count": 2311, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 406084, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:04:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.071, "last_frame_pushed_at": "2026-07-02T08:04:45+09:00", "last_frame_read_age_sec": 0.059, "last_frame_read_at": "2026-07-02T08:04:45+09:00", "mjpeg": {"age_sec": 0.056, "exists": true, "mtime": "2026-07-02T08:04:45.620234+09:00", "mtime_epoch": 1782947085.6202338, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203327}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:04:45+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5551.59, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 249, "pushed_frames": 249, "read_frames": 249, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.315, "pushed": 8.315, "read": 8.315}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 13.997, "count": 249, "max_ms": 15.935, "p95_ms": 14.824}, "gstreamer_push": {"avg_ms": 1.268, "count": 249, "max_ms": 1.428, "p95_ms": 1.337}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.598, "count": 249, "max_ms": 73.391, "p95_ms": 67.702}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.77, "count": 249, "max_ms": 91.324, "p95_ms": 87.099}, "yolo11_640_fire_11class": {"avg_ms": 71.489, "count": 249, "max_ms": 79.209, "p95_ms": 77.364}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.027, "count": 249, "max_ms": 84.726, "p95_ms": 78.87}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.295, "count": 249, "max_ms": 114.487, "p95_ms": 109.27}, "read_interval": {"avg_ms": 120.455, "count": 249, "max_ms": 129.515, "p95_ms": 124.747}, "triton_wait": {"avg_ms": 82.782, "count": 249, "max_ms": 91.336, "p95_ms": 87.112}, "visualize": {"avg_ms": 3.554, "count": 249, "max_ms": 4.198, "p95_ms": 3.748}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.945, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3384}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9494.031, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T08:04:45+09:00", "last_error_summary": "[h264 @ 0x55ace968be00] error while decoding MB 3 39, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 277, "h264_decode_error_count": 510, "h264_decode_error_count_per_hour": 62, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:04:45+09:00", "last_error_summary": "[h264 @ 0x55ace968be00] error while decoding MB 3 39, bytestream -7", "last_exit_at": "2026-07-02T08:04:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 277.163, "last_started_at": "2026-07-02T08:00:08+09:00", "pid": 49025, "process": {"exists": false, "pid": 49025}, "restart_count": 309, "restart_count_per_hour": 12, "rtsp_timeout_count": 120, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 310}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 128.912, "sample_count": 2312, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 406085, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:04:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T08:04:45+09:00", "last_frame_read_age_sec": 0.116, "last_frame_read_at": "2026-07-02T08:04:45+09:00", "mjpeg": {"age_sec": 0.114, "exists": true, "mtime": "2026-07-02T08:04:45.620234+09:00", "mtime_epoch": 1782947085.6202338, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203327}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:04:45+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5551.59, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 249, "pushed_frames": 249, "read_frames": 249, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.304, "pushed": 8.304, "read": 8.304}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 13.997, "count": 249, "max_ms": 15.935, "p95_ms": 14.824}, "gstreamer_push": {"avg_ms": 1.268, "count": 249, "max_ms": 1.428, "p95_ms": 1.337}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.595, "count": 249, "max_ms": 73.391, "p95_ms": 67.702}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.77, "count": 249, "max_ms": 91.324, "p95_ms": 87.099}, "yolo11_640_fire_11class": {"avg_ms": 71.478, "count": 249, "max_ms": 79.209, "p95_ms": 77.364}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.004, "count": 249, "max_ms": 84.726, "p95_ms": 78.87}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.294, "count": 249, "max_ms": 114.487, "p95_ms": 109.27}, "read_interval": {"avg_ms": 120.455, "count": 249, "max_ms": 129.515, "p95_ms": 124.747}, "triton_wait": {"avg_ms": 82.783, "count": 249, "max_ms": 91.336, "p95_ms": 87.112}, "visualize": {"avg_ms": 3.552, "count": 249, "max_ms": 4.198, "p95_ms": 3.748}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.986, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3384}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9494.207, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 277, "h264_decode_error_count": 510, "h264_decode_error_count_per_hour": 62, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:04:45+09:00", "last_error_summary": "[h264 @ 0x55ace968be00] error while decoding MB 3 39, bytestream -7", "last_exit_at": "2026-07-02T08:04:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 277.172, "last_started_at": "2026-07-02T08:00:08+09:00", "pid": 49025, "process": {"exists": false, "pid": 49025}, "restart_count": 309, "restart_count_per_hour": 12, "rtsp_timeout_count": 120, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 310}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 128.912, "sample_count": 2312, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 406085, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:04:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-02T08:04:45+09:00", "last_frame_read_age_sec": 0.125, "last_frame_read_at": "2026-07-02T08:04:45+09:00", "mjpeg": {"age_sec": 0.122, "exists": true, "mtime": "2026-07-02T08:04:45.620234+09:00", "mtime_epoch": 1782947085.6202338, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203327}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:04:45+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5551.59, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 249, "pushed_frames": 249, "read_frames": 249, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.301, "pushed": 8.301, "read": 8.301}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 13.997, "count": 249, "max_ms": 15.935, "p95_ms": 14.824}, "gstreamer_push": {"avg_ms": 1.268, "count": 249, "max_ms": 1.428, "p95_ms": 1.337}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.595, "count": 249, "max_ms": 73.391, "p95_ms": 67.702}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.77, "count": 249, "max_ms": 91.324, "p95_ms": 87.099}, "yolo11_640_fire_11class": {"avg_ms": 71.478, "count": 249, "max_ms": 79.209, "p95_ms": 77.364}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.004, "count": 249, "max_ms": 84.726, "p95_ms": 78.87}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.294, "count": 249, "max_ms": 114.487, "p95_ms": 109.27}, "read_interval": {"avg_ms": 120.455, "count": 249, "max_ms": 129.515, "p95_ms": 124.747}, "triton_wait": {"avg_ms": 82.783, "count": 249, "max_ms": 91.336, "p95_ms": 87.112}, "visualize": {"avg_ms": 3.552, "count": 249, "max_ms": 4.198, "p95_ms": 3.748}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3384}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9596.805, "limit_mb": 8796093022207.996}, "detail": {"pid": 49132, "restart_count": 310, "start_count": 311}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 277, "h264_decode_error_count": 510, "h264_decode_error_count_per_hour": 62, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:04:45+09:00", "last_error_summary": "[h264 @ 0x55ace968be00] error while decoding MB 3 39, bytestream -7", "last_exit_at": "2026-07-02T08:04:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T08:04:48+09:00", "pid": 49132, "process": {"exists": true, "fd_count": 5, "hwm_mb": 134.156, "name": "ffmpeg", "pid": 49132, "rss_mb": 134.156, "state": "S (sleeping)", "threads": 106}, "restart_count": 310, "restart_count_per_hour": 13, "rtsp_timeout_count": 120, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 311}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 128.912, "sample_count": 2312, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 406085, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:04:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.485, "last_frame_pushed_at": "2026-07-02T08:04:45+09:00", "last_frame_read_age_sec": 4.594, "last_frame_read_at": "2026-07-02T08:04:45+09:00", "mjpeg": {"age_sec": 4.468, "exists": true, "mtime": "2026-07-02T08:04:45.744235+09:00", "mtime_epoch": 1782947085.7442346, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203003}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:04:50+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5551.59, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 212, "pushed_frames": 212, "read_frames": 212, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.069, "pushed": 7.069, "read": 7.069}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.004, "count": 212, "max_ms": 15.935, "p95_ms": 14.824}, "gstreamer_push": {"avg_ms": 1.269, "count": 212, "max_ms": 1.428, "p95_ms": 1.335}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.426, "count": 212, "max_ms": 70.637, "p95_ms": 67.694}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.739, "count": 212, "max_ms": 91.324, "p95_ms": 87.505}, "yolo11_640_fire_11class": {"avg_ms": 71.398, "count": 212, "max_ms": 79.209, "p95_ms": 77.364}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.793, "count": 212, "max_ms": 84.726, "p95_ms": 78.913}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.26, "count": 212, "max_ms": 114.487, "p95_ms": 109.82}, "read_interval": {"avg_ms": 120.356, "count": 212, "max_ms": 129.515, "p95_ms": 125.21}, "triton_wait": {"avg_ms": 82.752, "count": 212, "max_ms": 91.336, "p95_ms": 87.517}, "visualize": {"avg_ms": 3.549, "count": 212, "max_ms": 4.198, "p95_ms": 3.742}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 4810}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9463.172, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56337c4c9d80] error while decoding MB 117 57, bytestream -13", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 76, "h264_decode_error_count": 76, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:09:48+09:00", "last_error_summary": "[h264 @ 0x56337c4c9d80] error while decoding MB 117 57, bytestream -13", "last_exit_at": "2026-07-02T07:57:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 729.29, "last_started_at": "2026-07-02T07:57:39+09:00", "pid": 48813, "process": {"exists": false, "pid": 48813}, "restart_count": 75, "restart_count_per_hour": 6, "rtsp_timeout_count": 136, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 76}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.387, "sample_count": 1957, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 577276, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:09:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-02T08:09:48+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-02T08:09:48+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-02T08:09:48.406901+09:00", "mtime_epoch": 1782947388.4069014, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 249454}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T08:09:48+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1139.996, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 205, "pushed_frames": 205, "read_frames": 205, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.838, "pushed": 6.838, "read": 6.838}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.661, "count": 204, "max_ms": 16.485, "p95_ms": 11.363}, "gstreamer_push": {"avg_ms": 1.279, "count": 205, "max_ms": 5.934, "p95_ms": 2.667}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.107, "count": 204, "max_ms": 75.661, "p95_ms": 69.297}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.662, "count": 204, "max_ms": 94.68, "p95_ms": 90.944}, "yolo11_640_fire_11class": {"avg_ms": 61.866, "count": 204, "max_ms": 84.442, "p95_ms": 80.844}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.683, "count": 204, "max_ms": 90.247, "p95_ms": 83.872}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.847, "count": 205, "max_ms": 227.862, "p95_ms": 180.655}, "read_interval": {"avg_ms": 146.676, "count": 205, "max_ms": 240.626, "p95_ms": 192.48}, "triton_wait": {"avg_ms": 70.925, "count": 204, "max_ms": 94.692, "p95_ms": 90.955}, "visualize": {"avg_ms": 4.908, "count": 204, "max_ms": 8.88, "p95_ms": 6.327}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.978, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 4810}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9462.363, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T08:09:48+09:00", "last_error_summary": "[h264 @ 0x56337c4c9d80] error while decoding MB 117 57, bytestream -13", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 76, "h264_decode_error_count": 76, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:09:48+09:00", "last_error_summary": "[h264 @ 0x56337c4c9d80] error while decoding MB 117 57, bytestream -13", "last_exit_at": "2026-07-02T08:09:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 729.391, "last_started_at": "2026-07-02T07:57:39+09:00", "pid": 48813, "process": {"exists": false, "pid": 48813}, "restart_count": 75, "restart_count_per_hour": 6, "rtsp_timeout_count": 136, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 76}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.387, "sample_count": 1957, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 577277, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:09:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T08:09:48+09:00", "last_frame_read_age_sec": 0.152, "last_frame_read_at": "2026-07-02T08:09:48+09:00", "mjpeg": {"age_sec": 0.143, "exists": true, "mtime": "2026-07-02T08:09:48.406901+09:00", "mtime_epoch": 1782947388.4069014, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 249454}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T08:09:48+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1139.996, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 205, "pushed_frames": 205, "read_frames": 204, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.857, "pushed": 6.857, "read": 6.824}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.67, "count": 205, "max_ms": 16.485, "p95_ms": 11.363}, "gstreamer_push": {"avg_ms": 1.279, "count": 205, "max_ms": 5.934, "p95_ms": 2.667}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.149, "count": 205, "max_ms": 75.661, "p95_ms": 69.297}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.627, "count": 205, "max_ms": 94.68, "p95_ms": 90.944}, "yolo11_640_fire_11class": {"avg_ms": 61.855, "count": 205, "max_ms": 84.442, "p95_ms": 80.844}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.677, "count": 205, "max_ms": 90.247, "p95_ms": 83.872}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.861, "count": 205, "max_ms": 227.862, "p95_ms": 180.655}, "read_interval": {"avg_ms": 146.634, "count": 204, "max_ms": 240.626, "p95_ms": 192.48}, "triton_wait": {"avg_ms": 70.889, "count": 205, "max_ms": 94.692, "p95_ms": 90.955}, "visualize": {"avg_ms": 4.907, "count": 205, "max_ms": 8.88, "p95_ms": 6.327}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.896, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 4810}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9462.195, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 76, "h264_decode_error_count": 76, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:09:48+09:00", "last_error_summary": "[h264 @ 0x56337c4c9d80] error while decoding MB 117 57, bytestream -13", "last_exit_at": "2026-07-02T08:09:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 729.395, "last_started_at": "2026-07-02T07:57:39+09:00", "pid": 48813, "process": {"exists": false, "pid": 48813}, "restart_count": 75, "restart_count_per_hour": 6, "rtsp_timeout_count": 136, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 76}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.387, "sample_count": 1957, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 577277, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:09:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-02T08:09:48+09:00", "last_frame_read_age_sec": 0.156, "last_frame_read_at": "2026-07-02T08:09:48+09:00", "mjpeg": {"age_sec": 0.147, "exists": true, "mtime": "2026-07-02T08:09:48.406901+09:00", "mtime_epoch": 1782947388.4069014, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 249454}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T08:09:48+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1139.996, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 205, "pushed_frames": 205, "read_frames": 204, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.856, "pushed": 6.856, "read": 6.823}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.67, "count": 205, "max_ms": 16.485, "p95_ms": 11.363}, "gstreamer_push": {"avg_ms": 1.279, "count": 205, "max_ms": 5.934, "p95_ms": 2.667}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.149, "count": 205, "max_ms": 75.661, "p95_ms": 69.297}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.627, "count": 205, "max_ms": 94.68, "p95_ms": 90.944}, "yolo11_640_fire_11class": {"avg_ms": 61.855, "count": 205, "max_ms": 84.442, "p95_ms": 80.844}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.677, "count": 205, "max_ms": 90.247, "p95_ms": 83.872}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.861, "count": 205, "max_ms": 227.862, "p95_ms": 180.655}, "read_interval": {"avg_ms": 146.634, "count": 204, "max_ms": 240.626, "p95_ms": 192.48}, "triton_wait": {"avg_ms": 70.889, "count": 205, "max_ms": 94.692, "p95_ms": 90.955}, "visualize": {"avg_ms": 4.907, "count": 205, "max_ms": 8.88, "p95_ms": 6.327}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.9, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 4810}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9577.414, "limit_mb": 8796093022207.996}, "detail": {"pid": 49239, "restart_count": 76, "start_count": 77}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 76, "h264_decode_error_count": 76, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:09:48+09:00", "last_error_summary": "[h264 @ 0x56337c4c9d80] error while decoding MB 117 57, bytestream -13", "last_exit_at": "2026-07-02T08:09:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T08:09:51+09:00", "pid": 49239, "process": {"exists": true, "fd_count": 5, "hwm_mb": 147.23, "name": "ffmpeg", "pid": 49239, "rss_mb": 147.23, "state": "R (running)", "threads": 49}, "restart_count": 76, "restart_count_per_hour": 7, "rtsp_timeout_count": 136, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 77}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.407, "sample_count": 1913, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 577277, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:09:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.511, "last_frame_pushed_at": "2026-07-02T08:09:48+09:00", "last_frame_read_age_sec": 4.658, "last_frame_read_at": "2026-07-02T08:09:48+09:00", "mjpeg": {"age_sec": 4.492, "exists": true, "mtime": "2026-07-02T08:09:48.562903+09:00", "mtime_epoch": 1782947388.5629032, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 246168}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T08:09:53+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1139.996, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 178, "pushed_frames": 178, "read_frames": 177, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.952, "pushed": 5.952, "read": 5.919}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.476, "count": 178, "max_ms": 16.485, "p95_ms": 11.435}, "gstreamer_push": {"avg_ms": 1.315, "count": 178, "max_ms": 5.934, "p95_ms": 3.008}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.369, "count": 178, "max_ms": 75.661, "p95_ms": 69.297}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 71.066, "count": 178, "max_ms": 94.68, "p95_ms": 91.19}, "yolo11_640_fire_11class": {"avg_ms": 62.389, "count": 178, "max_ms": 84.442, "p95_ms": 81.351}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.827, "count": 178, "max_ms": 90.247, "p95_ms": 84.224}}, "truncated_count": 0}, "processing_total": {"avg_ms": 130.634, "count": 178, "max_ms": 227.862, "p95_ms": 187.618}, "read_interval": {"avg_ms": 143.605, "count": 177, "max_ms": 240.626, "p95_ms": 197.862}, "triton_wait": {"avg_ms": 71.304, "count": 178, "max_ms": 94.692, "p95_ms": 91.203}, "visualize": {"avg_ms": 4.872, "count": 178, "max_ms": 8.88, "p95_ms": 6.446}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.905, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3407}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9495.664, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55a63d3c7e00] left block unavailable for requested intra mode [h264 @ 0x55a63d3c7e00] error while decoding MB 0 13, bytestream 1029 [h264 @ 0x55a63e0823c0] error while decoding MB 59 11, bytestream -5 [h264 @ 0x55a63dfb3540] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a63dfb3540] error while decoding MB 0 29, bytestream 3201 [in#0/rtsp @ 0x55a63d37fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a63d37fe00] Error retrieving a packet from demuxer:", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 278, "h264_decode_error_count": 513, "h264_decode_error_count_per_hour": 57, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:10:14+09:00", "last_error_summary": "[h264 @ 0x55a63d3c7e00] left block unavailable for requested intra mode [h264 @ 0x55a63d3c7e00] error while decoding MB 0 13, bytestream 1029 [h264 @ 0x55a63e0823c0] error while decoding MB 59 11, bytestream -5 [h264 @ 0x55a63dfb3540] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a63dfb3540] error while decoding MB 0 29, bytestream 3201 [in#0/rtsp @ 0x55a63d37fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a63d37fe00] Error retrieving a packet from demuxer:", "last_exit_at": "2026-07-02T08:04:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 326.608, "last_started_at": "2026-07-02T08:04:48+09:00", "pid": 49132, "process": {"exists": false, "pid": 49132}, "restart_count": 310, "restart_count_per_hour": 12, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 311}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.342, "sample_count": 2581, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 408867, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:10:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.065, "last_frame_pushed_at": "2026-07-02T08:10:14+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-02T08:10:14+09:00", "mjpeg": {"age_sec": 0.048, "exists": true, "mtime": "2026-07-02T08:10:14.771186+09:00", "mtime_epoch": 1782947414.7711856, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 207242}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:10:14+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5552.879, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 110, "pushed_frames": 110, "read_frames": 110, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.672, "pushed": 3.672, "read": 3.672}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 13.996, "count": 110, "max_ms": 16.646, "p95_ms": 15.979}, "gstreamer_push": {"avg_ms": 1.279, "count": 110, "max_ms": 1.586, "p95_ms": 1.357}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.019, "count": 110, "max_ms": 71.042, "p95_ms": 58.933}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.889, "count": 110, "max_ms": 91.8, "p95_ms": 87.812}, "yolo11_640_fire_11class": {"avg_ms": 51.83, "count": 110, "max_ms": 76.841, "p95_ms": 66.898}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.746, "count": 110, "max_ms": 88.005, "p95_ms": 81.237}}, "truncated_count": 0}, "processing_total": {"avg_ms": 83.306, "count": 110, "max_ms": 115.48, "p95_ms": 110.14}, "read_interval": {"avg_ms": 272.567, "count": 110, "max_ms": 19185.354, "p95_ms": 127.057}, "triton_wait": {"avg_ms": 60.902, "count": 110, "max_ms": 91.812, "p95_ms": 87.826}, "visualize": {"avg_ms": 3.601, "count": 110, "max_ms": 5.242, "p95_ms": 3.997}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.953, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3407}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9495.34, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T08:10:14+09:00", "last_error_summary": "[h264 @ 0x55a63d3c7e00] left block unavailable for requested intra mode [h264 @ 0x55a63d3c7e00] error while decoding MB 0 13, bytestream 1029 [h264 @ 0x55a63e0823c0] error while decoding MB 59 11, bytestream -5 [h264 @ 0x55a63dfb3540] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a63dfb3540] error while decoding MB 0 29, bytestream 3201 [in#0/rtsp @ 0x55a63d37fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a63d37fe00] Error retrieving a packet from demuxer:", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 278, "h264_decode_error_count": 513, "h264_decode_error_count_per_hour": 57, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:10:14+09:00", "last_error_summary": "[h264 @ 0x55a63d3c7e00] left block unavailable for requested intra mode [h264 @ 0x55a63d3c7e00] error while decoding MB 0 13, bytestream 1029 [h264 @ 0x55a63e0823c0] error while decoding MB 59 11, bytestream -5 [h264 @ 0x55a63dfb3540] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a63dfb3540] error while decoding MB 0 29, bytestream 3201 [in#0/rtsp @ 0x55a63d37fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a63d37fe00] Error retrieving a packet from demuxer:", "last_exit_at": "2026-07-02T08:10:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 326.648, "last_started_at": "2026-07-02T08:04:48+09:00", "pid": 49132, "process": {"exists": false, "pid": 49132}, "restart_count": 310, "restart_count_per_hour": 12, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 311}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.342, "sample_count": 2581, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 408868, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:10:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T08:10:14+09:00", "last_frame_read_age_sec": 0.093, "last_frame_read_at": "2026-07-02T08:10:14+09:00", "mjpeg": {"age_sec": 0.087, "exists": true, "mtime": "2026-07-02T08:10:14.771186+09:00", "mtime_epoch": 1782947414.7711856, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 207242}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:10:14+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5552.879, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 110, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.701, "pushed": 3.701, "read": 3.668}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 13.998, "count": 111, "max_ms": 16.646, "p95_ms": 15.979}, "gstreamer_push": {"avg_ms": 1.279, "count": 111, "max_ms": 1.586, "p95_ms": 1.357}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.011, "count": 111, "max_ms": 71.042, "p95_ms": 58.933}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.879, "count": 111, "max_ms": 91.8, "p95_ms": 87.812}, "yolo11_640_fire_11class": {"avg_ms": 51.821, "count": 111, "max_ms": 76.841, "p95_ms": 66.898}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.724, "count": 111, "max_ms": 88.005, "p95_ms": 81.237}}, "truncated_count": 0}, "processing_total": {"avg_ms": 83.319, "count": 111, "max_ms": 115.48, "p95_ms": 110.14}, "read_interval": {"avg_ms": 272.567, "count": 110, "max_ms": 19185.354, "p95_ms": 127.057}, "triton_wait": {"avg_ms": 60.892, "count": 111, "max_ms": 91.812, "p95_ms": 87.826}, "visualize": {"avg_ms": 3.616, "count": 111, "max_ms": 5.283, "p95_ms": 4.082}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3407}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9495.453, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 278, "h264_decode_error_count": 513, "h264_decode_error_count_per_hour": 57, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:10:14+09:00", "last_error_summary": "[h264 @ 0x55a63d3c7e00] left block unavailable for requested intra mode [h264 @ 0x55a63d3c7e00] error while decoding MB 0 13, bytestream 1029 [h264 @ 0x55a63e0823c0] error while decoding MB 59 11, bytestream -5 [h264 @ 0x55a63dfb3540] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a63dfb3540] error while decoding MB 0 29, bytestream 3201 [in#0/rtsp @ 0x55a63d37fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a63d37fe00] Error retrieving a packet from demuxer:", "last_exit_at": "2026-07-02T08:10:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 326.656, "last_started_at": "2026-07-02T08:04:48+09:00", "pid": 49132, "process": {"exists": false, "pid": 49132}, "restart_count": 310, "restart_count_per_hour": 12, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 311}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.342, "sample_count": 2581, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 408868, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:10:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-02T08:10:14+09:00", "last_frame_read_age_sec": 0.101, "last_frame_read_at": "2026-07-02T08:10:14+09:00", "mjpeg": {"age_sec": 0.095, "exists": true, "mtime": "2026-07-02T08:10:14.771186+09:00", "mtime_epoch": 1782947414.7711856, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 207242}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:10:14+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5552.879, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 110, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.7, "pushed": 3.7, "read": 3.667}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.0, "count": 110, "max_ms": 16.646, "p95_ms": 15.979}, "gstreamer_push": {"avg_ms": 1.279, "count": 111, "max_ms": 1.586, "p95_ms": 1.357}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.067, "count": 110, "max_ms": 71.042, "p95_ms": 58.933}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.871, "count": 110, "max_ms": 91.8, "p95_ms": 87.812}, "yolo11_640_fire_11class": {"avg_ms": 51.817, "count": 110, "max_ms": 76.841, "p95_ms": 66.898}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.723, "count": 110, "max_ms": 88.005, "p95_ms": 81.237}}, "truncated_count": 0}, "processing_total": {"avg_ms": 83.319, "count": 111, "max_ms": 115.48, "p95_ms": 110.14}, "read_interval": {"avg_ms": 272.567, "count": 110, "max_ms": 19185.354, "p95_ms": 127.057}, "triton_wait": {"avg_ms": 60.883, "count": 110, "max_ms": 91.812, "p95_ms": 87.826}, "visualize": {"avg_ms": 3.616, "count": 110, "max_ms": 5.283, "p95_ms": 4.082}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3407}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9600.121, "limit_mb": 8796093022207.996}, "detail": {"pid": 49391, "restart_count": 311, "start_count": 312}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 278, "h264_decode_error_count": 513, "h264_decode_error_count_per_hour": 57, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:10:14+09:00", "last_error_summary": "[h264 @ 0x55a63d3c7e00] left block unavailable for requested intra mode [h264 @ 0x55a63d3c7e00] error while decoding MB 0 13, bytestream 1029 [h264 @ 0x55a63e0823c0] error while decoding MB 59 11, bytestream -5 [h264 @ 0x55a63dfb3540] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55a63dfb3540] error while decoding MB 0 29, bytestream 3201 [in#0/rtsp @ 0x55a63d37fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a63d37fe00] Error retrieving a packet from demuxer:", "last_exit_at": "2026-07-02T08:10:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T08:10:17+09:00", "pid": 49391, "process": {"exists": true, "fd_count": 5, "hwm_mb": 134.008, "name": "ffmpeg", "pid": 49391, "rss_mb": 134.008, "state": "S (sleeping)", "threads": 106}, "restart_count": 311, "restart_count_per_hour": 13, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 312}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 115.158, "sample_count": 2544, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 408868, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:10:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.617, "last_frame_pushed_at": "2026-07-02T08:10:14+09:00", "last_frame_read_age_sec": 4.704, "last_frame_read_at": "2026-07-02T08:10:14+09:00", "mjpeg": {"age_sec": 4.598, "exists": true, "mtime": "2026-07-02T08:10:14.871187+09:00", "mtime_epoch": 1782947414.8711867, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 207934}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:10:19+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5552.879, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 65, "pushed_frames": 65, "read_frames": 65, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.167, "pushed": 2.167, "read": 2.167}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 13.836, "count": 65, "max_ms": 16.646, "p95_ms": 15.614}, "gstreamer_push": {"avg_ms": 1.281, "count": 65, "max_ms": 1.586, "p95_ms": 1.357}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.548, "count": 65, "max_ms": 71.042, "p95_ms": 58.933}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.106, "count": 65, "max_ms": 91.8, "p95_ms": 87.739}, "yolo11_640_fire_11class": {"avg_ms": 50.721, "count": 65, "max_ms": 76.841, "p95_ms": 66.898}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.308, "count": 65, "max_ms": 88.005, "p95_ms": 81.237}}, "truncated_count": 0}, "processing_total": {"avg_ms": 81.469, "count": 65, "max_ms": 115.48, "p95_ms": 110.401}, "read_interval": {"avg_ms": 390.538, "count": 65, "max_ms": 19185.354, "p95_ms": 130.323}, "triton_wait": {"avg_ms": 59.118, "count": 65, "max_ms": 91.812, "p95_ms": 87.751}, "visualize": {"avg_ms": 3.657, "count": 65, "max_ms": 5.283, "p95_ms": 4.733}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 4817}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9440.48, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5568605a7a80] error while decoding MB 83 29, bytestream -7 [in#0/rtsp @ 0x5568604c9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5568604c9e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 77, "h264_decode_error_count": 77, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:11:55+09:00", "last_error_summary": "[h264 @ 0x5568605a7a80] error while decoding MB 83 29, bytestream -7 [in#0/rtsp @ 0x5568604c9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5568604c9e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T08:09:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.089, "last_started_at": "2026-07-02T08:09:51+09:00", "pid": 49239, "process": {"exists": false, "pid": 49239}, "restart_count": 76, "restart_count_per_hour": 7, "rtsp_timeout_count": 138, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 77}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 173.59, "sample_count": 1861, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 578065, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:11:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.079, "last_frame_pushed_at": "2026-07-02T08:11:55+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-02T08:11:55+09:00", "mjpeg": {"age_sec": 0.063, "exists": true, "mtime": "2026-07-02T08:11:55.080324+09:00", "mtime_epoch": 1782947515.0803244, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 259564}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T08:11:55+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1139.824, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.688, "pushed": 0.688, "read": 0.745}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.937, "count": 12, "max_ms": 8.587, "p95_ms": 8.587}, "gstreamer_push": {"avg_ms": 1.336, "count": 12, "max_ms": 1.407, "p95_ms": 1.407}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.267, "count": 12, "max_ms": 70.347, "p95_ms": 70.347}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.825, "count": 12, "max_ms": 88.846, "p95_ms": 88.846}, "yolo11_640_fire_11class": {"avg_ms": 62.148, "count": 12, "max_ms": 71.921, "p95_ms": 71.921}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.31, "count": 12, "max_ms": 76.889, "p95_ms": 76.889}}, "truncated_count": 0}, "processing_total": {"avg_ms": 88.607, "count": 12, "max_ms": 103.126, "p95_ms": 103.126}, "read_interval": {"avg_ms": 2343.731, "count": 13, "max_ms": 16241.854, "p95_ms": 16241.854}, "triton_wait": {"avg_ms": 73.896, "count": 12, "max_ms": 88.858, "p95_ms": 88.858}, "visualize": {"avg_ms": 3.688, "count": 12, "max_ms": 4.021, "p95_ms": 4.021}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.449, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 4817}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9439.816, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T08:11:55+09:00", "last_error_summary": "[h264 @ 0x5568605a7a80] error while decoding MB 83 29, bytestream -7 [in#0/rtsp @ 0x5568604c9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5568604c9e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 77, "h264_decode_error_count": 77, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:11:55+09:00", "last_error_summary": "[h264 @ 0x5568605a7a80] error while decoding MB 83 29, bytestream -7 [in#0/rtsp @ 0x5568604c9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5568604c9e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T08:11:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.132, "last_started_at": "2026-07-02T08:09:51+09:00", "pid": 49239, "process": {"exists": false, "pid": 49239}, "restart_count": 76, "restart_count_per_hour": 7, "rtsp_timeout_count": 138, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 77}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 173.59, "sample_count": 1862, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 578066, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:11:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T08:11:55+09:00", "last_frame_read_age_sec": 0.108, "last_frame_read_at": "2026-07-02T08:11:55+09:00", "mjpeg": {"age_sec": 0.106, "exists": true, "mtime": "2026-07-02T08:11:55.080324+09:00", "mtime_epoch": 1782947515.0803244, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 259564}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T08:11:55+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1139.824, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.743, "pushed": 0.743, "read": 0.743}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.915, "count": 13, "max_ms": 8.587, "p95_ms": 8.587}, "gstreamer_push": {"avg_ms": 1.335, "count": 13, "max_ms": 1.407, "p95_ms": 1.407}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.916, "count": 13, "max_ms": 70.347, "p95_ms": 70.347}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.786, "count": 13, "max_ms": 88.846, "p95_ms": 88.846}, "yolo11_640_fire_11class": {"avg_ms": 61.613, "count": 13, "max_ms": 71.921, "p95_ms": 71.921}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.784, "count": 13, "max_ms": 76.889, "p95_ms": 76.889}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.534, "count": 13, "max_ms": 103.126, "p95_ms": 103.126}, "read_interval": {"avg_ms": 2343.731, "count": 13, "max_ms": 16241.854, "p95_ms": 16241.854}, "triton_wait": {"avg_ms": 74.853, "count": 13, "max_ms": 88.858, "p95_ms": 88.858}, "visualize": {"avg_ms": 3.687, "count": 13, "max_ms": 4.021, "p95_ms": 4.021}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.491, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 4817}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9440.047, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 77, "h264_decode_error_count": 77, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:11:55+09:00", "last_error_summary": "[h264 @ 0x5568605a7a80] error while decoding MB 83 29, bytestream -7 [in#0/rtsp @ 0x5568604c9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5568604c9e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T08:11:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.139, "last_started_at": "2026-07-02T08:09:51+09:00", "pid": 49239, "process": {"exists": false, "pid": 49239}, "restart_count": 76, "restart_count_per_hour": 7, "rtsp_timeout_count": 138, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 77}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 173.59, "sample_count": 1862, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 578066, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:11:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-02T08:11:55+09:00", "last_frame_read_age_sec": 0.114, "last_frame_read_at": "2026-07-02T08:11:55+09:00", "mjpeg": {"age_sec": 0.112, "exists": true, "mtime": "2026-07-02T08:11:55.080324+09:00", "mtime_epoch": 1782947515.0803244, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 259564}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T08:11:55+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1139.824, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.743, "pushed": 0.743, "read": 0.743}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.915, "count": 13, "max_ms": 8.587, "p95_ms": 8.587}, "gstreamer_push": {"avg_ms": 1.335, "count": 13, "max_ms": 1.407, "p95_ms": 1.407}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.916, "count": 13, "max_ms": 70.347, "p95_ms": 70.347}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.786, "count": 13, "max_ms": 88.846, "p95_ms": 88.846}, "yolo11_640_fire_11class": {"avg_ms": 61.613, "count": 13, "max_ms": 71.921, "p95_ms": 71.921}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.784, "count": 13, "max_ms": 76.889, "p95_ms": 76.889}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.534, "count": 13, "max_ms": 103.126, "p95_ms": 103.126}, "read_interval": {"avg_ms": 2343.731, "count": 13, "max_ms": 16241.854, "p95_ms": 16241.854}, "triton_wait": {"avg_ms": 74.853, "count": 13, "max_ms": 88.858, "p95_ms": 88.858}, "visualize": {"avg_ms": 3.687, "count": 13, "max_ms": 4.021, "p95_ms": 4.021}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.498, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 4817}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9622.07, "limit_mb": 8796093022207.996}, "detail": {"pid": 49498, "restart_count": 77, "start_count": 78}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 77, "h264_decode_error_count": 77, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:11:55+09:00", "last_error_summary": "[h264 @ 0x5568605a7a80] error while decoding MB 83 29, bytestream -7 [in#0/rtsp @ 0x5568604c9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5568604c9e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T08:11:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T08:11:57+09:00", "pid": 49498, "process": {"exists": true, "fd_count": 5, "hwm_mb": 210.148, "name": "ffmpeg", "pid": 49498, "rss_mb": 210.148, "state": "S (sleeping)", "threads": 106}, "restart_count": 77, "restart_count_per_hour": 8, "rtsp_timeout_count": 138, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 78}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 173.59, "sample_count": 1835, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 578066, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:11:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.372, "last_frame_pushed_at": "2026-07-02T08:11:55+09:00", "last_frame_read_age_sec": 4.475, "last_frame_read_at": "2026-07-02T08:11:55+09:00", "mjpeg": {"age_sec": 4.357, "exists": true, "mtime": "2026-07-02T08:11:55.196326+09:00", "mtime_epoch": 1782947515.1963258, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 259507}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T08:11:59+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1139.824, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.595, "pushed": 0.595, "read": 0.595}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.915, "count": 13, "max_ms": 8.587, "p95_ms": 8.587}, "gstreamer_push": {"avg_ms": 1.335, "count": 13, "max_ms": 1.407, "p95_ms": 1.407}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.916, "count": 13, "max_ms": 70.347, "p95_ms": 70.347}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.786, "count": 13, "max_ms": 88.846, "p95_ms": 88.846}, "yolo11_640_fire_11class": {"avg_ms": 61.613, "count": 13, "max_ms": 71.921, "p95_ms": 71.921}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.784, "count": 13, "max_ms": 76.889, "p95_ms": 76.889}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.534, "count": 13, "max_ms": 103.126, "p95_ms": 103.126}, "read_interval": {"avg_ms": 2343.731, "count": 13, "max_ms": 16241.854, "p95_ms": 16241.854}, "triton_wait": {"avg_ms": 74.853, "count": 13, "max_ms": 88.858, "p95_ms": 88.858}, "visualize": {"avg_ms": 3.687, "count": 13, "max_ms": 4.021, "p95_ms": 4.021}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.859, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3436}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9501.176, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x560fc4807240] error while decoding MB 77 38, bytestream -28", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 279, "h264_decode_error_count": 514, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:16:55+09:00", "last_error_summary": "[h264 @ 0x560fc4807240] error while decoding MB 77 38, bytestream -28", "last_exit_at": "2026-07-02T08:10:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 398.26, "last_started_at": "2026-07-02T08:10:17+09:00", "pid": 49391, "process": {"exists": false, "pid": 49391}, "restart_count": 311, "restart_count_per_hour": 11, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 312}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 108.649, "sample_count": 2550, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 412330, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:16:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.069, "last_frame_pushed_at": "2026-07-02T08:16:55+09:00", "last_frame_read_age_sec": 0.056, "last_frame_read_at": "2026-07-02T08:16:55+09:00", "mjpeg": {"age_sec": 0.053, "exists": true, "mtime": "2026-07-02T08:16:55.676151+09:00", "mtime_epoch": 1782947815.6761508, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205966}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:16:55+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5554.426, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 253, "pushed_frames": 253, "read_frames": 253, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.435, "pushed": 8.435, "read": 8.435}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.009, "count": 253, "max_ms": 18.062, "p95_ms": 14.752}, "gstreamer_push": {"avg_ms": 1.261, "count": 253, "max_ms": 1.424, "p95_ms": 1.336}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.622, "count": 253, "max_ms": 77.262, "p95_ms": 68.485}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.676, "count": 253, "max_ms": 92.852, "p95_ms": 86.057}, "yolo11_640_fire_11class": {"avg_ms": 65.902, "count": 253, "max_ms": 79.606, "p95_ms": 72.581}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.295, "count": 253, "max_ms": 84.302, "p95_ms": 78.932}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.143, "count": 253, "max_ms": 115.253, "p95_ms": 108.598}, "read_interval": {"avg_ms": 118.751, "count": 253, "max_ms": 130.799, "p95_ms": 124.825}, "triton_wait": {"avg_ms": 80.688, "count": 253, "max_ms": 92.865, "p95_ms": 86.07}, "visualize": {"avg_ms": 3.575, "count": 253, "max_ms": 4.821, "p95_ms": 3.887}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3436}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9500.551, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T08:16:55+09:00", "last_error_summary": "[h264 @ 0x560fc4807240] error while decoding MB 77 38, bytestream -28", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 279, "h264_decode_error_count": 514, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:16:55+09:00", "last_error_summary": "[h264 @ 0x560fc4807240] error while decoding MB 77 38, bytestream -28", "last_exit_at": "2026-07-02T08:16:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 398.322, "last_started_at": "2026-07-02T08:10:17+09:00", "pid": 49391, "process": {"exists": false, "pid": 49391}, "restart_count": 311, "restart_count_per_hour": 11, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 312}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 108.649, "sample_count": 2550, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 412331, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:16:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T08:16:55+09:00", "last_frame_read_age_sec": 0.118, "last_frame_read_at": "2026-07-02T08:16:55+09:00", "mjpeg": {"age_sec": 0.114, "exists": true, "mtime": "2026-07-02T08:16:55.676151+09:00", "mtime_epoch": 1782947815.6761508, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205966}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:16:55+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5554.426, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 253, "pushed_frames": 253, "read_frames": 252, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.452, "pushed": 8.452, "read": 8.418}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.008, "count": 253, "max_ms": 18.062, "p95_ms": 14.752}, "gstreamer_push": {"avg_ms": 1.261, "count": 253, "max_ms": 1.424, "p95_ms": 1.333}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.636, "count": 253, "max_ms": 77.262, "p95_ms": 68.485}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.692, "count": 253, "max_ms": 92.852, "p95_ms": 86.139}, "yolo11_640_fire_11class": {"avg_ms": 65.919, "count": 253, "max_ms": 79.606, "p95_ms": 72.581}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.319, "count": 253, "max_ms": 84.302, "p95_ms": 78.932}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.158, "count": 253, "max_ms": 115.253, "p95_ms": 108.598}, "read_interval": {"avg_ms": 118.742, "count": 252, "max_ms": 130.799, "p95_ms": 124.825}, "triton_wait": {"avg_ms": 80.704, "count": 253, "max_ms": 92.865, "p95_ms": 86.152}, "visualize": {"avg_ms": 3.575, "count": 253, "max_ms": 4.821, "p95_ms": 3.887}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.935, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3436}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9500.645, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 279, "h264_decode_error_count": 514, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:16:55+09:00", "last_error_summary": "[h264 @ 0x560fc4807240] error while decoding MB 77 38, bytestream -28", "last_exit_at": "2026-07-02T08:16:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 398.331, "last_started_at": "2026-07-02T08:10:17+09:00", "pid": 49391, "process": {"exists": false, "pid": 49391}, "restart_count": 311, "restart_count_per_hour": 11, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 312}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 108.649, "sample_count": 2550, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 412331, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:16:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-02T08:16:55+09:00", "last_frame_read_age_sec": 0.126, "last_frame_read_at": "2026-07-02T08:16:55+09:00", "mjpeg": {"age_sec": 0.123, "exists": true, "mtime": "2026-07-02T08:16:55.676151+09:00", "mtime_epoch": 1782947815.6761508, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205966}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:16:55+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5554.426, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 253, "pushed_frames": 253, "read_frames": 252, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.449, "pushed": 8.449, "read": 8.416}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.008, "count": 253, "max_ms": 18.062, "p95_ms": 14.752}, "gstreamer_push": {"avg_ms": 1.261, "count": 253, "max_ms": 1.424, "p95_ms": 1.333}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.636, "count": 253, "max_ms": 77.262, "p95_ms": 68.485}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.692, "count": 253, "max_ms": 92.852, "p95_ms": 86.139}, "yolo11_640_fire_11class": {"avg_ms": 65.919, "count": 253, "max_ms": 79.606, "p95_ms": 72.581}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.319, "count": 253, "max_ms": 84.302, "p95_ms": 78.932}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.158, "count": 253, "max_ms": 115.253, "p95_ms": 108.598}, "read_interval": {"avg_ms": 118.742, "count": 252, "max_ms": 130.799, "p95_ms": 124.825}, "triton_wait": {"avg_ms": 80.704, "count": 253, "max_ms": 92.865, "p95_ms": 86.152}, "visualize": {"avg_ms": 3.575, "count": 253, "max_ms": 4.821, "p95_ms": 3.887}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.944, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3436}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9611.094, "limit_mb": 8796093022207.996}, "detail": {"pid": 49650, "restart_count": 312, "start_count": 313}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 279, "h264_decode_error_count": 514, "h264_decode_error_count_per_hour": 48, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:16:55+09:00", "last_error_summary": "[h264 @ 0x560fc4807240] error while decoding MB 77 38, bytestream -28", "last_exit_at": "2026-07-02T08:16:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T08:16:57+09:00", "pid": 49650, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.93, "name": "ffmpeg", "pid": 49650, "rss_mb": 138.93, "state": "S (sleeping)", "threads": 106}, "restart_count": 312, "restart_count_per_hour": 12, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 313}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 108.679, "sample_count": 2507, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 412331, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:16:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.051, "last_frame_pushed_at": "2026-07-02T08:16:55+09:00", "last_frame_read_age_sec": 4.161, "last_frame_read_at": "2026-07-02T08:16:55+09:00", "mjpeg": {"age_sec": 4.038, "exists": true, "mtime": "2026-07-02T08:16:55.796152+09:00", "mtime_epoch": 1782947815.7961524, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206230}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:16:59+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5554.426, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 219, "pushed_frames": 219, "read_frames": 219, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.303, "pushed": 7.303, "read": 7.303}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.003, "count": 219, "max_ms": 18.062, "p95_ms": 14.661}, "gstreamer_push": {"avg_ms": 1.262, "count": 219, "max_ms": 1.424, "p95_ms": 1.333}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.686, "count": 219, "max_ms": 77.262, "p95_ms": 68.503}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.464, "count": 219, "max_ms": 92.852, "p95_ms": 86.277}, "yolo11_640_fire_11class": {"avg_ms": 66.136, "count": 219, "max_ms": 79.606, "p95_ms": 72.992}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.429, "count": 219, "max_ms": 84.302, "p95_ms": 78.982}}, "truncated_count": 0}, "processing_total": {"avg_ms": 102.914, "count": 219, "max_ms": 115.253, "p95_ms": 109.03}, "read_interval": {"avg_ms": 118.492, "count": 219, "max_ms": 130.799, "p95_ms": 124.825}, "triton_wait": {"avg_ms": 80.476, "count": 219, "max_ms": 92.865, "p95_ms": 86.289}, "visualize": {"avg_ms": 3.569, "count": 219, "max_ms": 4.821, "p95_ms": 3.858}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.987, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 4851}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9449.961, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x561beca57700] error while decoding MB 99 67, bytestream -9 [in#0/rtsp @ 0x561beca35e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561beca35e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 78, "h264_decode_error_count": 78, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:20:23+09:00", "last_error_summary": "[h264 @ 0x561beca57700] error while decoding MB 99 67, bytestream -9 [in#0/rtsp @ 0x561beca35e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561beca35e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T08:11:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 506.255, "last_started_at": "2026-07-02T08:11:57+09:00", "pid": 49498, "process": {"exists": false, "pid": 49498}, "restart_count": 77, "restart_count_per_hour": 7, "rtsp_timeout_count": 140, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 78}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.703, "sample_count": 2333, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 582164, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:20:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.079, "last_frame_pushed_at": "2026-07-02T08:20:23+09:00", "last_frame_read_age_sec": 0.066, "last_frame_read_at": "2026-07-02T08:20:23+09:00", "mjpeg": {"age_sec": 0.06, "exists": true, "mtime": "2026-07-02T08:20:23.747046+09:00", "mtime_epoch": 1782948023.747046, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 249928}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T08:20:23+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1141.688, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 256, "pushed_frames": 256, "read_frames": 257, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.536, "pushed": 8.536, "read": 8.569}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.781, "count": 256, "max_ms": 8.735, "p95_ms": 6.281}, "gstreamer_push": {"avg_ms": 1.652, "count": 256, "max_ms": 6.127, "p95_ms": 3.791}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.289, "count": 256, "max_ms": 78.254, "p95_ms": 63.01}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 86.867, "count": 256, "max_ms": 97.294, "p95_ms": 93.474}, "yolo11_640_fire_11class": {"avg_ms": 68.371, "count": 256, "max_ms": 78.804, "p95_ms": 76.159}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.477, "count": 256, "max_ms": 86.133, "p95_ms": 83.074}}, "truncated_count": 0}, "processing_total": {"avg_ms": 101.533, "count": 256, "max_ms": 111.672, "p95_ms": 108.289}, "read_interval": {"avg_ms": 116.79, "count": 257, "max_ms": 128.003, "p95_ms": 124.342}, "triton_wait": {"avg_ms": 86.88, "count": 256, "max_ms": 97.308, "p95_ms": 93.487}, "visualize": {"avg_ms": 3.691, "count": 256, "max_ms": 4.608, "p95_ms": 4.025}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 4851}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9449.68, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T08:20:23+09:00", "last_error_summary": "[h264 @ 0x561beca57700] error while decoding MB 99 67, bytestream -9 [in#0/rtsp @ 0x561beca35e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561beca35e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 78, "h264_decode_error_count": 78, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:20:23+09:00", "last_error_summary": "[h264 @ 0x561beca57700] error while decoding MB 99 67, bytestream -9 [in#0/rtsp @ 0x561beca35e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561beca35e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T08:20:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 506.28, "last_started_at": "2026-07-02T08:11:57+09:00", "pid": 49498, "process": {"exists": false, "pid": 49498}, "restart_count": 77, "restart_count_per_hour": 7, "rtsp_timeout_count": 140, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 78}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.703, "sample_count": 2334, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 582165, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:20:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-02T08:20:23+09:00", "last_frame_read_age_sec": 0.091, "last_frame_read_at": "2026-07-02T08:20:23+09:00", "mjpeg": {"age_sec": 0.086, "exists": true, "mtime": "2026-07-02T08:20:23.747046+09:00", "mtime_epoch": 1782948023.747046, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 249928}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T08:20:23+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1141.688, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 257, "pushed_frames": 257, "read_frames": 256, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.59, "pushed": 8.59, "read": 8.556}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.782, "count": 257, "max_ms": 8.735, "p95_ms": 6.281}, "gstreamer_push": {"avg_ms": 1.652, "count": 257, "max_ms": 6.127, "p95_ms": 3.791}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.262, "count": 257, "max_ms": 78.254, "p95_ms": 63.01}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 86.757, "count": 257, "max_ms": 97.294, "p95_ms": 93.474}, "yolo11_640_fire_11class": {"avg_ms": 68.307, "count": 257, "max_ms": 78.804, "p95_ms": 76.159}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.397, "count": 257, "max_ms": 86.133, "p95_ms": 83.074}}, "truncated_count": 0}, "processing_total": {"avg_ms": 101.452, "count": 257, "max_ms": 111.672, "p95_ms": 108.289}, "read_interval": {"avg_ms": 116.897, "count": 256, "max_ms": 128.003, "p95_ms": 124.342}, "triton_wait": {"avg_ms": 86.769, "count": 257, "max_ms": 97.308, "p95_ms": 93.487}, "visualize": {"avg_ms": 3.709, "count": 257, "max_ms": 8.345, "p95_ms": 4.031}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.92, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 4851}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9449.594, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 78, "h264_decode_error_count": 78, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:20:23+09:00", "last_error_summary": "[h264 @ 0x561beca57700] error while decoding MB 99 67, bytestream -9 [in#0/rtsp @ 0x561beca35e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561beca35e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T08:20:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 506.289, "last_started_at": "2026-07-02T08:11:57+09:00", "pid": 49498, "process": {"exists": false, "pid": 49498}, "restart_count": 77, "restart_count_per_hour": 7, "rtsp_timeout_count": 140, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 78}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.703, "sample_count": 2334, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 582165, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:20:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.017, "last_frame_pushed_at": "2026-07-02T08:20:23+09:00", "last_frame_read_age_sec": 0.1, "last_frame_read_at": "2026-07-02T08:20:23+09:00", "mjpeg": {"age_sec": 0.095, "exists": true, "mtime": "2026-07-02T08:20:23.747046+09:00", "mtime_epoch": 1782948023.747046, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 249928}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T08:20:23+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1141.688, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 257, "pushed_frames": 257, "read_frames": 256, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.587, "pushed": 8.587, "read": 8.554}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.782, "count": 257, "max_ms": 8.735, "p95_ms": 6.281}, "gstreamer_push": {"avg_ms": 1.652, "count": 257, "max_ms": 6.127, "p95_ms": 3.791}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.262, "count": 257, "max_ms": 78.254, "p95_ms": 63.01}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 86.757, "count": 257, "max_ms": 97.294, "p95_ms": 93.474}, "yolo11_640_fire_11class": {"avg_ms": 68.307, "count": 257, "max_ms": 78.804, "p95_ms": 76.159}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.397, "count": 257, "max_ms": 86.133, "p95_ms": 83.074}}, "truncated_count": 0}, "processing_total": {"avg_ms": 101.452, "count": 257, "max_ms": 111.672, "p95_ms": 108.289}, "read_interval": {"avg_ms": 116.897, "count": 256, "max_ms": 128.003, "p95_ms": 124.342}, "triton_wait": {"avg_ms": 86.769, "count": 257, "max_ms": 97.308, "p95_ms": 93.487}, "visualize": {"avg_ms": 3.709, "count": 257, "max_ms": 8.345, "p95_ms": 4.031}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.929, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 4851}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9527.473, "limit_mb": 8796093022207.996}, "detail": {"pid": 49757, "restart_count": 78, "start_count": 79}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 78, "h264_decode_error_count": 78, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:20:23+09:00", "last_error_summary": "[h264 @ 0x561beca57700] error while decoding MB 99 67, bytestream -9 [in#0/rtsp @ 0x561beca35e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561beca35e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T08:20:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T08:20:25+09:00", "pid": 49757, "process": {"exists": true, "fd_count": 5, "hwm_mb": 110.156, "name": "ffmpeg", "pid": 49757, "rss_mb": 110.156, "state": "S (sleeping)", "threads": 19}, "restart_count": 78, "restart_count_per_hour": 8, "rtsp_timeout_count": 140, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 79}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.694, "sample_count": 2300, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 582165, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:20:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.153, "last_frame_pushed_at": "2026-07-02T08:20:23+09:00", "last_frame_read_age_sec": 4.236, "last_frame_read_at": "2026-07-02T08:20:23+09:00", "mjpeg": {"age_sec": 4.135, "exists": true, "mtime": "2026-07-02T08:20:23.843047+09:00", "mtime_epoch": 1782948023.8430474, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 249928}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T08:20:27+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1141.688, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 221, "pushed_frames": 221, "read_frames": 220, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.383, "pushed": 7.383, "read": 7.35}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.78, "count": 221, "max_ms": 8.735, "p95_ms": 6.233}, "gstreamer_push": {"avg_ms": 1.67, "count": 221, "max_ms": 6.127, "p95_ms": 3.791}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.286, "count": 221, "max_ms": 78.254, "p95_ms": 62.212}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 86.936, "count": 221, "max_ms": 97.294, "p95_ms": 93.692}, "yolo11_640_fire_11class": {"avg_ms": 68.492, "count": 221, "max_ms": 78.804, "p95_ms": 76.048}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.601, "count": 221, "max_ms": 86.133, "p95_ms": 83.117}}, "truncated_count": 0}, "processing_total": {"avg_ms": 101.651, "count": 221, "max_ms": 111.672, "p95_ms": 108.307}, "read_interval": {"avg_ms": 117.27, "count": 220, "max_ms": 128.003, "p95_ms": 124.355}, "triton_wait": {"avg_ms": 86.949, "count": 221, "max_ms": 97.308, "p95_ms": 93.705}, "visualize": {"avg_ms": 3.714, "count": 221, "max_ms": 8.345, "p95_ms": 4.031}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.933, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3457}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9498.957, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x563969c7bc80] cabac decode of qscale diff failed at 6 6 [h264 @ 0x563969c7bc80] error while decoding MB 6 6, bytestream 541 [h264 @ 0x5639693d5a40] error while decoding MB 50 17, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 280, "h264_decode_error_count": 517, "h264_decode_error_count_per_hour": 50, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:22:07+09:00", "last_error_summary": "[h264 @ 0x563969c7bc80] cabac decode of qscale diff failed at 6 6 [h264 @ 0x563969c7bc80] error while decoding MB 6 6, bytestream 541 [h264 @ 0x5639693d5a40] error while decoding MB 50 17, bytestream -9", "last_exit_at": "2026-07-02T08:16:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 309.827, "last_started_at": "2026-07-02T08:16:57+09:00", "pid": 49650, "process": {"exists": false, "pid": 49650}, "restart_count": 312, "restart_count_per_hour": 11, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 313}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 131.032, "sample_count": 2522, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 414917, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:22:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.069, "last_frame_pushed_at": "2026-07-02T08:22:07+09:00", "last_frame_read_age_sec": 0.056, "last_frame_read_at": "2026-07-02T08:22:07+09:00", "mjpeg": {"age_sec": 0.051, "exists": true, "mtime": "2026-07-02T08:22:07.608541+09:00", "mtime_epoch": 1782948127.608541, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 208311}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:22:07+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5555.715, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 252, "pushed_frames": 252, "read_frames": 252, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.418, "pushed": 8.418, "read": 8.418}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 13.977, "count": 252, "max_ms": 19.344, "p95_ms": 14.372}, "gstreamer_push": {"avg_ms": 1.261, "count": 252, "max_ms": 2.275, "p95_ms": 1.34}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.677, "count": 252, "max_ms": 73.595, "p95_ms": 67.49}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.426, "count": 252, "max_ms": 93.43, "p95_ms": 86.298}, "yolo11_640_fire_11class": {"avg_ms": 65.811, "count": 252, "max_ms": 77.589, "p95_ms": 73.042}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.968, "count": 252, "max_ms": 84.629, "p95_ms": 79.62}}, "truncated_count": 0}, "processing_total": {"avg_ms": 102.831, "count": 252, "max_ms": 115.851, "p95_ms": 108.722}, "read_interval": {"avg_ms": 118.969, "count": 252, "max_ms": 134.615, "p95_ms": 125.651}, "triton_wait": {"avg_ms": 80.438, "count": 252, "max_ms": 93.442, "p95_ms": 86.311}, "visualize": {"avg_ms": 3.567, "count": 252, "max_ms": 5.649, "p95_ms": 3.817}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.935, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3457}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9497.797, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T08:22:07+09:00", "last_error_summary": "[h264 @ 0x563969c7bc80] cabac decode of qscale diff failed at 6 6 [h264 @ 0x563969c7bc80] error while decoding MB 6 6, bytestream 541 [h264 @ 0x5639693d5a40] error while decoding MB 50 17, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 280, "h264_decode_error_count": 517, "h264_decode_error_count_per_hour": 50, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:22:07+09:00", "last_error_summary": "[h264 @ 0x563969c7bc80] cabac decode of qscale diff failed at 6 6 [h264 @ 0x563969c7bc80] error while decoding MB 6 6, bytestream 541 [h264 @ 0x5639693d5a40] error while decoding MB 50 17, bytestream -9", "last_exit_at": "2026-07-02T08:22:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 309.887, "last_started_at": "2026-07-02T08:16:57+09:00", "pid": 49650, "process": {"exists": false, "pid": 49650}, "restart_count": 312, "restart_count_per_hour": 11, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 313}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 131.032, "sample_count": 2522, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 414918, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:22:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T08:22:07+09:00", "last_frame_read_age_sec": 0.116, "last_frame_read_at": "2026-07-02T08:22:07+09:00", "mjpeg": {"age_sec": 0.112, "exists": true, "mtime": "2026-07-02T08:22:07.608541+09:00", "mtime_epoch": 1782948127.608541, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 208311}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:22:07+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5555.715, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 253, "pushed_frames": 253, "read_frames": 252, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.435, "pushed": 8.435, "read": 8.401}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 13.977, "count": 253, "max_ms": 19.344, "p95_ms": 14.372}, "gstreamer_push": {"avg_ms": 1.261, "count": 253, "max_ms": 2.275, "p95_ms": 1.34}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.702, "count": 253, "max_ms": 73.595, "p95_ms": 67.699}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.444, "count": 253, "max_ms": 93.43, "p95_ms": 86.298}, "yolo11_640_fire_11class": {"avg_ms": 65.835, "count": 253, "max_ms": 77.589, "p95_ms": 73.042}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.993, "count": 253, "max_ms": 84.629, "p95_ms": 79.62}}, "truncated_count": 0}, "processing_total": {"avg_ms": 102.847, "count": 253, "max_ms": 115.851, "p95_ms": 108.722}, "read_interval": {"avg_ms": 118.969, "count": 252, "max_ms": 134.615, "p95_ms": 125.651}, "triton_wait": {"avg_ms": 80.456, "count": 253, "max_ms": 93.442, "p95_ms": 86.311}, "visualize": {"avg_ms": 3.566, "count": 253, "max_ms": 5.649, "p95_ms": 3.817}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3457}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9497.668, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 280, "h264_decode_error_count": 517, "h264_decode_error_count_per_hour": 50, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:22:07+09:00", "last_error_summary": "[h264 @ 0x563969c7bc80] cabac decode of qscale diff failed at 6 6 [h264 @ 0x563969c7bc80] error while decoding MB 6 6, bytestream 541 [h264 @ 0x5639693d5a40] error while decoding MB 50 17, bytestream -9", "last_exit_at": "2026-07-02T08:22:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 309.896, "last_started_at": "2026-07-02T08:16:57+09:00", "pid": 49650, "process": {"exists": false, "pid": 49650}, "restart_count": 312, "restart_count_per_hour": 11, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 313}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 131.032, "sample_count": 2522, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 414918, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:22:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-02T08:22:07+09:00", "last_frame_read_age_sec": 0.125, "last_frame_read_at": "2026-07-02T08:22:07+09:00", "mjpeg": {"age_sec": 0.121, "exists": true, "mtime": "2026-07-02T08:22:07.608541+09:00", "mtime_epoch": 1782948127.608541, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 208311}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:22:07+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5555.715, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 252, "pushed_frames": 252, "read_frames": 252, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.403, "pushed": 8.403, "read": 8.403}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 13.976, "count": 252, "max_ms": 19.344, "p95_ms": 14.372}, "gstreamer_push": {"avg_ms": 1.261, "count": 252, "max_ms": 2.275, "p95_ms": 1.34}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.705, "count": 252, "max_ms": 73.595, "p95_ms": 67.699}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.451, "count": 252, "max_ms": 93.43, "p95_ms": 86.298}, "yolo11_640_fire_11class": {"avg_ms": 65.85, "count": 252, "max_ms": 77.589, "p95_ms": 73.042}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.995, "count": 252, "max_ms": 84.629, "p95_ms": 79.62}}, "truncated_count": 0}, "processing_total": {"avg_ms": 102.853, "count": 252, "max_ms": 115.851, "p95_ms": 108.722}, "read_interval": {"avg_ms": 118.969, "count": 252, "max_ms": 134.615, "p95_ms": 125.651}, "triton_wait": {"avg_ms": 80.463, "count": 252, "max_ms": 93.442, "p95_ms": 86.311}, "visualize": {"avg_ms": 3.566, "count": 252, "max_ms": 5.649, "p95_ms": 3.817}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.988, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3457}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9608.516, "limit_mb": 8796093022207.996}, "detail": {"pid": 49909, "restart_count": 313, "start_count": 314}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 280, "h264_decode_error_count": 517, "h264_decode_error_count_per_hour": 50, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:22:07+09:00", "last_error_summary": "[h264 @ 0x563969c7bc80] cabac decode of qscale diff failed at 6 6 [h264 @ 0x563969c7bc80] error while decoding MB 6 6, bytestream 541 [h264 @ 0x5639693d5a40] error while decoding MB 50 17, bytestream -9", "last_exit_at": "2026-07-02T08:22:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T08:22:10+09:00", "pid": 49909, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.191, "name": "ffmpeg", "pid": 49909, "rss_mb": 139.191, "state": "S (sleeping)", "threads": 106}, "restart_count": 313, "restart_count_per_hour": 12, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 314}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 132.074, "sample_count": 2482, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 414918, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:22:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "h264_decode_error_count_per_hour>=50", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.838, "last_frame_pushed_at": "2026-07-02T08:22:07+09:00", "last_frame_read_age_sec": 4.947, "last_frame_read_at": "2026-07-02T08:22:07+09:00", "mjpeg": {"age_sec": 4.823, "exists": true, "mtime": "2026-07-02T08:22:07.728543+09:00", "mtime_epoch": 1782948127.7285428, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 208660}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:22:12+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5555.715, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 212, "pushed_frames": 212, "read_frames": 211, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.073, "pushed": 7.073, "read": 7.04}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 13.994, "count": 212, "max_ms": 19.344, "p95_ms": 14.56}, "gstreamer_push": {"avg_ms": 1.265, "count": 212, "max_ms": 2.275, "p95_ms": 1.342}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.526, "count": 212, "max_ms": 73.055, "p95_ms": 67.699}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.463, "count": 212, "max_ms": 93.43, "p95_ms": 86.449}, "yolo11_640_fire_11class": {"avg_ms": 65.968, "count": 212, "max_ms": 77.589, "p95_ms": 73.047}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.051, "count": 212, "max_ms": 84.629, "p95_ms": 80.015}}, "truncated_count": 0}, "processing_total": {"avg_ms": 102.9, "count": 212, "max_ms": 115.851, "p95_ms": 108.852}, "read_interval": {"avg_ms": 119.073, "count": 211, "max_ms": 134.615, "p95_ms": 126.466}, "triton_wait": {"avg_ms": 80.475, "count": 212, "max_ms": 93.442, "p95_ms": 86.461}, "visualize": {"avg_ms": 3.575, "count": 212, "max_ms": 5.649, "p95_ms": 3.845}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.973, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 4862}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9455.035, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55e9c2a57540] error while decoding MB 82 30, bytestream -13 [in#0/rtsp @ 0x55e9c2a4be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e9c2a4be00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 79, "h264_decode_error_count": 79, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:23:31+09:00", "last_error_summary": "[h264 @ 0x55e9c2a57540] error while decoding MB 82 30, bytestream -13 [in#0/rtsp @ 0x55e9c2a4be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e9c2a4be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T08:20:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 185.967, "last_started_at": "2026-07-02T08:20:25+09:00", "pid": 49757, "process": {"exists": false, "pid": 49757}, "restart_count": 78, "restart_count_per_hour": 7, "rtsp_timeout_count": 142, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 79}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.236, "sample_count": 2120, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 583553, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:23:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.078, "last_frame_pushed_at": "2026-07-02T08:23:31+09:00", "last_frame_read_age_sec": 0.063, "last_frame_read_at": "2026-07-02T08:23:31+09:00", "mjpeg": {"age_sec": 0.058, "exists": true, "mtime": "2026-07-02T08:23:31.885773+09:00", "mtime_epoch": 1782948211.8857732, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 263882}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T08:23:31+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1139.566, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 64, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.134, "pushed": 2.134, "read": 2.134}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.71, "count": 64, "max_ms": 8.574, "p95_ms": 7.03}, "gstreamer_push": {"avg_ms": 1.73, "count": 64, "max_ms": 6.465, "p95_ms": 4.368}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.049, "count": 64, "max_ms": 76.476, "p95_ms": 68.636}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 85.585, "count": 64, "max_ms": 91.921, "p95_ms": 91.159}, "yolo11_640_fire_11class": {"avg_ms": 72.602, "count": 64, "max_ms": 84.184, "p95_ms": 82.183}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.241, "count": 64, "max_ms": 86.251, "p95_ms": 83.79}}, "truncated_count": 0}, "processing_total": {"avg_ms": 100.349, "count": 64, "max_ms": 106.219, "p95_ms": 105.778}, "read_interval": {"avg_ms": 469.186, "count": 64, "max_ms": 13838.28, "p95_ms": 124.525}, "triton_wait": {"avg_ms": 85.598, "count": 64, "max_ms": 91.931, "p95_ms": 91.173}, "visualize": {"avg_ms": 3.743, "count": 64, "max_ms": 4.562, "p95_ms": 3.987}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.988, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 4862}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9454.637, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T08:23:31+09:00", "last_error_summary": "[h264 @ 0x55e9c2a57540] error while decoding MB 82 30, bytestream -13 [in#0/rtsp @ 0x55e9c2a4be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e9c2a4be00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 79, "h264_decode_error_count": 79, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:23:31+09:00", "last_error_summary": "[h264 @ 0x55e9c2a57540] error while decoding MB 82 30, bytestream -13 [in#0/rtsp @ 0x55e9c2a4be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e9c2a4be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T08:23:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 186.014, "last_started_at": "2026-07-02T08:20:25+09:00", "pid": 49757, "process": {"exists": false, "pid": 49757}, "restart_count": 78, "restart_count_per_hour": 7, "rtsp_timeout_count": 142, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 79}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.236, "sample_count": 2121, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 583554, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:23:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T08:23:31+09:00", "last_frame_read_age_sec": 0.109, "last_frame_read_at": "2026-07-02T08:23:31+09:00", "mjpeg": {"age_sec": 0.105, "exists": true, "mtime": "2026-07-02T08:23:31.885773+09:00", "mtime_epoch": 1782948211.8857732, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 263882}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T08:23:31+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1139.566, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 63, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.139, "pushed": 2.139, "read": 2.106}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.712, "count": 64, "max_ms": 8.574, "p95_ms": 7.03}, "gstreamer_push": {"avg_ms": 1.732, "count": 64, "max_ms": 6.465, "p95_ms": 4.368}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.047, "count": 64, "max_ms": 76.476, "p95_ms": 68.636}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 85.553, "count": 64, "max_ms": 91.921, "p95_ms": 91.159}, "yolo11_640_fire_11class": {"avg_ms": 72.539, "count": 64, "max_ms": 84.184, "p95_ms": 82.183}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.452, "count": 64, "max_ms": 86.251, "p95_ms": 83.79}}, "truncated_count": 0}, "processing_total": {"avg_ms": 100.319, "count": 64, "max_ms": 106.219, "p95_ms": 105.778}, "read_interval": {"avg_ms": 474.754, "count": 63, "max_ms": 13838.28, "p95_ms": 124.525}, "triton_wait": {"avg_ms": 85.566, "count": 64, "max_ms": 91.931, "p95_ms": 91.173}, "visualize": {"avg_ms": 3.743, "count": 64, "max_ms": 4.562, "p95_ms": 3.987}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.918, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 4862}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9454.777, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 79, "h264_decode_error_count": 79, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:23:31+09:00", "last_error_summary": "[h264 @ 0x55e9c2a57540] error while decoding MB 82 30, bytestream -13 [in#0/rtsp @ 0x55e9c2a4be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e9c2a4be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T08:23:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 186.024, "last_started_at": "2026-07-02T08:20:25+09:00", "pid": 49757, "process": {"exists": false, "pid": 49757}, "restart_count": 78, "restart_count_per_hour": 7, "rtsp_timeout_count": 142, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 79}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.236, "sample_count": 2121, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 583554, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:23:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-02T08:23:31+09:00", "last_frame_read_age_sec": 0.119, "last_frame_read_at": "2026-07-02T08:23:31+09:00", "mjpeg": {"age_sec": 0.115, "exists": true, "mtime": "2026-07-02T08:23:31.885773+09:00", "mtime_epoch": 1782948211.8857732, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 263882}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T08:23:32+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1139.566, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 63, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.138, "pushed": 2.138, "read": 2.105}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.712, "count": 64, "max_ms": 8.574, "p95_ms": 7.03}, "gstreamer_push": {"avg_ms": 1.732, "count": 64, "max_ms": 6.465, "p95_ms": 4.368}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.047, "count": 64, "max_ms": 76.476, "p95_ms": 68.636}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 85.553, "count": 64, "max_ms": 91.921, "p95_ms": 91.159}, "yolo11_640_fire_11class": {"avg_ms": 72.539, "count": 64, "max_ms": 84.184, "p95_ms": 82.183}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.452, "count": 64, "max_ms": 86.251, "p95_ms": 83.79}}, "truncated_count": 0}, "processing_total": {"avg_ms": 100.319, "count": 64, "max_ms": 106.219, "p95_ms": 105.778}, "read_interval": {"avg_ms": 474.754, "count": 63, "max_ms": 13838.28, "p95_ms": 124.525}, "triton_wait": {"avg_ms": 85.566, "count": 64, "max_ms": 91.931, "p95_ms": 91.173}, "visualize": {"avg_ms": 3.743, "count": 64, "max_ms": 4.562, "p95_ms": 3.987}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.928, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 4862}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9629.223, "limit_mb": 8796093022207.996}, "detail": {"pid": 50016, "restart_count": 79, "start_count": 80}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 79, "h264_decode_error_count": 79, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:23:31+09:00", "last_error_summary": "[h264 @ 0x55e9c2a57540] error while decoding MB 82 30, bytestream -13 [in#0/rtsp @ 0x55e9c2a4be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e9c2a4be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T08:23:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T08:23:34+09:00", "pid": 50016, "process": {"exists": true, "fd_count": 5, "hwm_mb": 204.812, "name": "ffmpeg", "pid": 50016, "rss_mb": 204.812, "state": "S (sleeping)", "threads": 106}, "restart_count": 79, "restart_count_per_hour": 8, "rtsp_timeout_count": 142, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 80}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.231, "sample_count": 2078, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 583554, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:23:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.322, "last_frame_pushed_at": "2026-07-02T08:23:31+09:00", "last_frame_read_age_sec": 4.425, "last_frame_read_at": "2026-07-02T08:23:31+09:00", "mjpeg": {"age_sec": 4.304, "exists": true, "mtime": "2026-07-02T08:23:32.001775+09:00", "mtime_epoch": 1782948212.0017748, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 263748}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T08:23:36+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1139.566, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 28, "pushed_frames": 28, "read_frames": 27, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.935, "pushed": 0.935, "read": 0.902}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.893, "count": 28, "max_ms": 8.574, "p95_ms": 8.564}, "gstreamer_push": {"avg_ms": 1.561, "count": 28, "max_ms": 4.368, "p95_ms": 4.227}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.488, "count": 28, "max_ms": 64.098, "p95_ms": 55.568}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.393, "count": 28, "max_ms": 91.68, "p95_ms": 91.159}, "yolo11_640_fire_11class": {"avg_ms": 69.856, "count": 28, "max_ms": 80.631, "p95_ms": 78.488}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.245, "count": 28, "max_ms": 86.251, "p95_ms": 83.79}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.283, "count": 28, "max_ms": 105.993, "p95_ms": 105.778}, "read_interval": {"avg_ms": 949.091, "count": 27, "max_ms": 13838.28, "p95_ms": 8933.101}, "triton_wait": {"avg_ms": 82.407, "count": 28, "max_ms": 91.694, "p95_ms": 91.173}, "visualize": {"avg_ms": 3.787, "count": 28, "max_ms": 4.562, "p95_ms": 4.404}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.95, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3481}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9512.352, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x555f7dda9a00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555f7dda9a00] error while decoding MB 0 37, bytestream 7575 [h264 @ 0x555f7df47580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555f7df47580] error while decoding MB 0 27, bytestream 722", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 281, "h264_decode_error_count": 519, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:27:52+09:00", "last_error_summary": "[h264 @ 0x555f7dda9a00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555f7dda9a00] error while decoding MB 0 37, bytestream 7575 [h264 @ 0x555f7df47580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555f7df47580] error while decoding MB 0 27, bytestream 722", "last_exit_at": "2026-07-02T08:22:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 341.702, "last_started_at": "2026-07-02T08:22:10+09:00", "pid": 49909, "process": {"exists": false, "pid": 49909}, "restart_count": 313, "restart_count_per_hour": 11, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 314}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 119.823, "sample_count": 2559, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 417810, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:27:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.068, "last_frame_pushed_at": "2026-07-02T08:27:52+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-02T08:27:52+09:00", "mjpeg": {"age_sec": 0.051, "exists": true, "mtime": "2026-07-02T08:27:52.201658+09:00", "mtime_epoch": 1782948472.2016585, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205144}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:27:52+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5557.004, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 250, "pushed_frames": 250, "read_frames": 250, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.345, "pushed": 8.345, "read": 8.345}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 13.964, "count": 250, "max_ms": 19.515, "p95_ms": 14.944}, "gstreamer_push": {"avg_ms": 1.257, "count": 250, "max_ms": 1.383, "p95_ms": 1.327}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.005, "count": 250, "max_ms": 73.445, "p95_ms": 67.783}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.76, "count": 250, "max_ms": 89.833, "p95_ms": 86.667}, "yolo11_640_fire_11class": {"avg_ms": 70.181, "count": 250, "max_ms": 80.198, "p95_ms": 76.112}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.602, "count": 250, "max_ms": 87.199, "p95_ms": 80.423}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.176, "count": 250, "max_ms": 112.465, "p95_ms": 109.766}, "read_interval": {"avg_ms": 120.025, "count": 250, "max_ms": 128.227, "p95_ms": 125.561}, "triton_wait": {"avg_ms": 81.772, "count": 250, "max_ms": 89.846, "p95_ms": 86.678}, "visualize": {"avg_ms": 3.584, "count": 250, "max_ms": 5.695, "p95_ms": 3.904}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.959, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3481}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9511.641, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T08:27:52+09:00", "last_error_summary": "[h264 @ 0x555f7dda9a00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555f7dda9a00] error while decoding MB 0 37, bytestream 7575 [h264 @ 0x555f7df47580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555f7df47580] error while decoding MB 0 27, bytestream 722", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 281, "h264_decode_error_count": 519, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:27:52+09:00", "last_error_summary": "[h264 @ 0x555f7dda9a00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555f7dda9a00] error while decoding MB 0 37, bytestream 7575 [h264 @ 0x555f7df47580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555f7df47580] error while decoding MB 0 27, bytestream 722", "last_exit_at": "2026-07-02T08:27:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 341.764, "last_started_at": "2026-07-02T08:22:10+09:00", "pid": 49909, "process": {"exists": false, "pid": 49909}, "restart_count": 313, "restart_count_per_hour": 11, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 314}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 119.823, "sample_count": 2559, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 417811, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:27:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-02T08:27:52+09:00", "last_frame_read_age_sec": 0.117, "last_frame_read_at": "2026-07-02T08:27:52+09:00", "mjpeg": {"age_sec": 0.113, "exists": true, "mtime": "2026-07-02T08:27:52.201658+09:00", "mtime_epoch": 1782948472.2016585, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205144}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:27:52+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5557.004, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 250, "pushed_frames": 250, "read_frames": 249, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.359, "pushed": 8.359, "read": 8.326}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 13.965, "count": 250, "max_ms": 19.515, "p95_ms": 14.944}, "gstreamer_push": {"avg_ms": 1.257, "count": 250, "max_ms": 1.383, "p95_ms": 1.327}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.02, "count": 250, "max_ms": 73.445, "p95_ms": 67.783}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.778, "count": 250, "max_ms": 89.833, "p95_ms": 86.667}, "yolo11_640_fire_11class": {"avg_ms": 70.191, "count": 250, "max_ms": 80.198, "p95_ms": 76.112}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.626, "count": 250, "max_ms": 87.199, "p95_ms": 80.423}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.195, "count": 250, "max_ms": 112.465, "p95_ms": 109.766}, "read_interval": {"avg_ms": 120.02, "count": 249, "max_ms": 128.227, "p95_ms": 125.561}, "triton_wait": {"avg_ms": 81.79, "count": 250, "max_ms": 89.846, "p95_ms": 86.678}, "visualize": {"avg_ms": 3.584, "count": 250, "max_ms": 5.695, "p95_ms": 3.904}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.907, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3481}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9511.688, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 281, "h264_decode_error_count": 519, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:27:52+09:00", "last_error_summary": "[h264 @ 0x555f7dda9a00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555f7dda9a00] error while decoding MB 0 37, bytestream 7575 [h264 @ 0x555f7df47580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555f7df47580] error while decoding MB 0 27, bytestream 722", "last_exit_at": "2026-07-02T08:27:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 341.773, "last_started_at": "2026-07-02T08:22:10+09:00", "pid": 49909, "process": {"exists": false, "pid": 49909}, "restart_count": 313, "restart_count_per_hour": 11, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 314}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 119.823, "sample_count": 2559, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 417811, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:27:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.017, "last_frame_pushed_at": "2026-07-02T08:27:52+09:00", "last_frame_read_age_sec": 0.126, "last_frame_read_at": "2026-07-02T08:27:52+09:00", "mjpeg": {"age_sec": 0.122, "exists": true, "mtime": "2026-07-02T08:27:52.201658+09:00", "mtime_epoch": 1782948472.2016585, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205144}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:27:52+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5557.004, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 250, "pushed_frames": 250, "read_frames": 249, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.357, "pushed": 8.357, "read": 8.323}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 13.965, "count": 250, "max_ms": 19.515, "p95_ms": 14.944}, "gstreamer_push": {"avg_ms": 1.257, "count": 250, "max_ms": 1.383, "p95_ms": 1.327}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.02, "count": 250, "max_ms": 73.445, "p95_ms": 67.783}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.778, "count": 250, "max_ms": 89.833, "p95_ms": 86.667}, "yolo11_640_fire_11class": {"avg_ms": 70.191, "count": 250, "max_ms": 80.198, "p95_ms": 76.112}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.626, "count": 250, "max_ms": 87.199, "p95_ms": 80.423}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.195, "count": 250, "max_ms": 112.465, "p95_ms": 109.766}, "read_interval": {"avg_ms": 120.02, "count": 249, "max_ms": 128.227, "p95_ms": 125.561}, "triton_wait": {"avg_ms": 81.79, "count": 250, "max_ms": 89.846, "p95_ms": 86.678}, "visualize": {"avg_ms": 3.584, "count": 250, "max_ms": 5.695, "p95_ms": 3.904}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.916, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3481}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9611.805, "limit_mb": 8796093022207.996}, "detail": {"pid": 50168, "restart_count": 314, "start_count": 315}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 281, "h264_decode_error_count": 519, "h264_decode_error_count_per_hour": 47, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:27:52+09:00", "last_error_summary": "[h264 @ 0x555f7dda9a00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555f7dda9a00] error while decoding MB 0 37, bytestream 7575 [h264 @ 0x555f7df47580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555f7df47580] error while decoding MB 0 27, bytestream 722", "last_exit_at": "2026-07-02T08:27:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T08:27:55+09:00", "pid": 50168, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.598, "name": "ffmpeg", "pid": 50168, "rss_mb": 136.598, "state": "S (sleeping)", "threads": 106}, "restart_count": 314, "restart_count_per_hour": 12, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 315}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 120.131, "sample_count": 2520, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 417811, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:27:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.734, "last_frame_pushed_at": "2026-07-02T08:27:52+09:00", "last_frame_read_age_sec": 4.843, "last_frame_read_at": "2026-07-02T08:27:52+09:00", "mjpeg": {"age_sec": 4.715, "exists": true, "mtime": "2026-07-02T08:27:52.325660+09:00", "mtime_epoch": 1782948472.3256602, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205093}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:27:57+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5557.004, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 211, "pushed_frames": 211, "read_frames": 210, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.035, "pushed": 7.035, "read": 7.002}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 13.965, "count": 211, "max_ms": 19.515, "p95_ms": 14.944}, "gstreamer_push": {"avg_ms": 1.256, "count": 211, "max_ms": 1.383, "p95_ms": 1.327}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.063, "count": 211, "max_ms": 73.445, "p95_ms": 67.783}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.069, "count": 211, "max_ms": 89.833, "p95_ms": 86.923}, "yolo11_640_fire_11class": {"avg_ms": 70.115, "count": 211, "max_ms": 80.198, "p95_ms": 76.008}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.788, "count": 211, "max_ms": 87.199, "p95_ms": 80.551}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.493, "count": 211, "max_ms": 112.465, "p95_ms": 110.033}, "read_interval": {"avg_ms": 120.244, "count": 210, "max_ms": 128.227, "p95_ms": 125.762}, "triton_wait": {"avg_ms": 82.081, "count": 211, "max_ms": 89.846, "p95_ms": 86.935}, "visualize": {"avg_ms": 3.595, "count": 211, "max_ms": 5.695, "p95_ms": 3.95}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3493}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9508.414, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T08:27:52+09:00", "last_error_summary": "[h264 @ 0x555f7dda9a00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555f7dda9a00] error while decoding MB 0 37, bytestream 7575 [h264 @ 0x555f7df47580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555f7df47580] error while decoding MB 0 27, bytestream 722", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 281, "h264_decode_error_count": 519, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:27:52+09:00", "last_error_summary": "[h264 @ 0x555f7dda9a00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555f7dda9a00] error while decoding MB 0 37, bytestream 7575 [h264 @ 0x555f7df47580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555f7df47580] error while decoding MB 0 27, bytestream 722", "last_exit_at": "2026-07-02T08:31:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 195.227, "last_started_at": "2026-07-02T08:27:55+09:00", "pid": 50168, "process": {"exists": false, "pid": 50168}, "restart_count": 314, "restart_count_per_hour": 11, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 315}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.931, "sample_count": 2304, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 419256, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:31:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T08:31:10+09:00", "last_frame_read_age_sec": 0.113, "last_frame_read_at": "2026-07-02T08:31:10+09:00", "mjpeg": {"age_sec": 0.11, "exists": true, "mtime": "2026-07-02T08:31:10.156670+09:00", "mtime_epoch": 1782948670.15667, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205330}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:31:10+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5557.777, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 251, "pushed_frames": 251, "read_frames": 250, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.391, "pushed": 8.391, "read": 8.357}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 13.981, "count": 251, "max_ms": 18.795, "p95_ms": 15.874}, "gstreamer_push": {"avg_ms": 1.247, "count": 251, "max_ms": 1.509, "p95_ms": 1.338}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.673, "count": 251, "max_ms": 73.125, "p95_ms": 68.462}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.987, "count": 251, "max_ms": 92.311, "p95_ms": 88.514}, "yolo11_640_fire_11class": {"avg_ms": 69.831, "count": 251, "max_ms": 82.951, "p95_ms": 77.311}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.353, "count": 251, "max_ms": 86.784, "p95_ms": 82.831}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.687, "count": 251, "max_ms": 147.432, "p95_ms": 112.58}, "read_interval": {"avg_ms": 119.797, "count": 250, "max_ms": 160.262, "p95_ms": 128.611}, "triton_wait": {"avg_ms": 81.038, "count": 251, "max_ms": 92.323, "p95_ms": 88.523}, "visualize": {"avg_ms": 3.545, "count": 251, "max_ms": 4.975, "p95_ms": 4.182}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.915, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3493}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9508.465, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 281, "h264_decode_error_count": 519, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:27:52+09:00", "last_error_summary": "[h264 @ 0x555f7dda9a00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555f7dda9a00] error while decoding MB 0 37, bytestream 7575 [h264 @ 0x555f7df47580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555f7df47580] error while decoding MB 0 27, bytestream 722", "last_exit_at": "2026-07-02T08:31:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 195.235, "last_started_at": "2026-07-02T08:27:55+09:00", "pid": 50168, "process": {"exists": false, "pid": 50168}, "restart_count": 314, "restart_count_per_hour": 11, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 315}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.931, "sample_count": 2304, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 419256, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:31:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-02T08:31:10+09:00", "last_frame_read_age_sec": 0.121, "last_frame_read_at": "2026-07-02T08:31:10+09:00", "mjpeg": {"age_sec": 0.118, "exists": true, "mtime": "2026-07-02T08:31:10.156670+09:00", "mtime_epoch": 1782948670.15667, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205330}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:31:10+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5557.777, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 251, "pushed_frames": 251, "read_frames": 250, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.388, "pushed": 8.388, "read": 8.355}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 13.981, "count": 251, "max_ms": 18.795, "p95_ms": 15.874}, "gstreamer_push": {"avg_ms": 1.247, "count": 251, "max_ms": 1.509, "p95_ms": 1.338}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.673, "count": 251, "max_ms": 73.125, "p95_ms": 68.462}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.987, "count": 251, "max_ms": 92.311, "p95_ms": 88.514}, "yolo11_640_fire_11class": {"avg_ms": 69.831, "count": 251, "max_ms": 82.951, "p95_ms": 77.311}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.353, "count": 251, "max_ms": 86.784, "p95_ms": 82.831}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.687, "count": 251, "max_ms": 147.432, "p95_ms": 112.58}, "read_interval": {"avg_ms": 119.797, "count": 250, "max_ms": 160.262, "p95_ms": 128.611}, "triton_wait": {"avg_ms": 81.038, "count": 251, "max_ms": 92.323, "p95_ms": 88.523}, "visualize": {"avg_ms": 3.545, "count": 251, "max_ms": 4.975, "p95_ms": 4.182}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.923, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3493}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9615.957, "limit_mb": 8796093022207.996}, "detail": {"pid": 50275, "restart_count": 315, "start_count": 316}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 281, "h264_decode_error_count": 519, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:27:52+09:00", "last_error_summary": "[h264 @ 0x555f7dda9a00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555f7dda9a00] error while decoding MB 0 37, bytestream 7575 [h264 @ 0x555f7df47580] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555f7df47580] error while decoding MB 0 27, bytestream 722", "last_exit_at": "2026-07-02T08:31:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T08:31:13+09:00", "pid": 50275, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.246, "name": "ffmpeg", "pid": 50275, "rss_mb": 139.246, "state": "S (sleeping)", "threads": 106}, "restart_count": 315, "restart_count_per_hour": 12, "rtsp_timeout_count": 122, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 316}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 146.077, "sample_count": 2261, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 419256, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:31:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.989, "last_frame_pushed_at": "2026-07-02T08:31:10+09:00", "last_frame_read_age_sec": 5.095, "last_frame_read_at": "2026-07-02T08:31:10+09:00", "mjpeg": {"age_sec": 4.972, "exists": true, "mtime": "2026-07-02T08:31:10.276672+09:00", "mtime_epoch": 1782948670.2766721, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205404}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:31:15+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5557.777, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 211, "pushed_frames": 211, "read_frames": 210, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.038, "pushed": 7.038, "read": 7.004}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 13.824, "count": 211, "max_ms": 18.795, "p95_ms": 14.676}, "gstreamer_push": {"avg_ms": 1.247, "count": 211, "max_ms": 1.417, "p95_ms": 1.33}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.36, "count": 211, "max_ms": 73.125, "p95_ms": 68.462}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.616, "count": 211, "max_ms": 91.288, "p95_ms": 88.514}, "yolo11_640_fire_11class": {"avg_ms": 70.427, "count": 211, "max_ms": 82.951, "p95_ms": 77.012}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.81, "count": 211, "max_ms": 86.784, "p95_ms": 82.712}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.751, "count": 211, "max_ms": 113.395, "p95_ms": 110.201}, "read_interval": {"avg_ms": 118.93, "count": 210, "max_ms": 131.388, "p95_ms": 126.058}, "triton_wait": {"avg_ms": 81.65, "count": 211, "max_ms": 91.297, "p95_ms": 88.523}, "visualize": {"avg_ms": 3.479, "count": 211, "max_ms": 4.274, "p95_ms": 3.764}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.981, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 4899}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9445.254, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x556d52f2d480] error while decoding MB 112 64, bytestream -10 [in#0/rtsp @ 0x556d52e15e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556d52e15e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 80, "h264_decode_error_count": 80, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:31:50+09:00", "last_error_summary": "[h264 @ 0x556d52f2d480] error while decoding MB 112 64, bytestream -10 [in#0/rtsp @ 0x556d52e15e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556d52e15e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T08:23:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 496.083, "last_started_at": "2026-07-02T08:23:34+09:00", "pid": 50016, "process": {"exists": false, "pid": 50016}, "restart_count": 79, "restart_count_per_hour": 6, "rtsp_timeout_count": 144, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 80}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.618, "sample_count": 2641, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 587882, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:31:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.078, "last_frame_pushed_at": "2026-07-02T08:31:50+09:00", "last_frame_read_age_sec": 0.066, "last_frame_read_at": "2026-07-02T08:31:50+09:00", "mjpeg": {"age_sec": 0.062, "exists": true, "mtime": "2026-07-02T08:31:50.325286+09:00", "mtime_epoch": 1782948710.3252857, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 257225}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T08:31:50+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1144.465, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 161, "pushed_frames": 161, "read_frames": 162, "select_timeouts": 12, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.938, "pushed": 8.938, "read": 8.994}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.036, "count": 161, "max_ms": 9.316, "p95_ms": 7.949}, "gstreamer_push": {"avg_ms": 1.578, "count": 161, "max_ms": 7.896, "p95_ms": 3.696}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.99, "count": 161, "max_ms": 79.384, "p95_ms": 64.595}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.008, "count": 161, "max_ms": 92.863, "p95_ms": 90.041}, "yolo11_640_fire_11class": {"avg_ms": 72.855, "count": 161, "max_ms": 84.892, "p95_ms": 82.949}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.067, "count": 161, "max_ms": 85.451, "p95_ms": 84.167}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.104, "count": 161, "max_ms": 108.394, "p95_ms": 105.481}, "read_interval": {"avg_ms": 243.188, "count": 162, "max_ms": 21449.95, "p95_ms": 122.232}, "triton_wait": {"avg_ms": 81.025, "count": 161, "max_ms": 92.877, "p95_ms": 90.054}, "visualize": {"avg_ms": 3.787, "count": 161, "max_ms": 5.937, "p95_ms": 4.479}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.013, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 4899}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9444.785, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T08:31:50+09:00", "last_error_summary": "[h264 @ 0x556d52f2d480] error while decoding MB 112 64, bytestream -10 [in#0/rtsp @ 0x556d52e15e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556d52e15e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 80, "h264_decode_error_count": 80, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:31:50+09:00", "last_error_summary": "[h264 @ 0x556d52f2d480] error while decoding MB 112 64, bytestream -10 [in#0/rtsp @ 0x556d52e15e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556d52e15e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T08:31:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 496.134, "last_started_at": "2026-07-02T08:23:34+09:00", "pid": 50016, "process": {"exists": false, "pid": 50016}, "restart_count": 79, "restart_count_per_hour": 6, "rtsp_timeout_count": 144, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 80}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.618, "sample_count": 2642, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 587883, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:31:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-02T08:31:50+09:00", "last_frame_read_age_sec": 0.117, "last_frame_read_at": "2026-07-02T08:31:50+09:00", "mjpeg": {"age_sec": 0.114, "exists": true, "mtime": "2026-07-02T08:31:50.325286+09:00", "mtime_epoch": 1782948710.3252857, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 257225}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T08:31:50+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1144.465, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 162, "pushed_frames": 162, "read_frames": 162, "select_timeouts": 12, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.968, "pushed": 8.968, "read": 8.968}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.034, "count": 162, "max_ms": 9.316, "p95_ms": 7.949}, "gstreamer_push": {"avg_ms": 1.577, "count": 162, "max_ms": 7.896, "p95_ms": 3.696}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.009, "count": 162, "max_ms": 79.384, "p95_ms": 64.595}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.066, "count": 162, "max_ms": 92.863, "p95_ms": 90.114}, "yolo11_640_fire_11class": {"avg_ms": 72.914, "count": 162, "max_ms": 84.892, "p95_ms": 82.949}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.129, "count": 162, "max_ms": 85.451, "p95_ms": 84.167}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.158, "count": 162, "max_ms": 108.394, "p95_ms": 105.481}, "read_interval": {"avg_ms": 243.188, "count": 162, "max_ms": 21449.95, "p95_ms": 122.232}, "triton_wait": {"avg_ms": 81.083, "count": 162, "max_ms": 92.877, "p95_ms": 90.127}, "visualize": {"avg_ms": 3.786, "count": 162, "max_ms": 5.937, "p95_ms": 4.479}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.064, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 4899}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9444.789, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 80, "h264_decode_error_count": 80, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:31:50+09:00", "last_error_summary": "[h264 @ 0x556d52f2d480] error while decoding MB 112 64, bytestream -10 [in#0/rtsp @ 0x556d52e15e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556d52e15e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T08:31:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 496.142, "last_started_at": "2026-07-02T08:23:34+09:00", "pid": 50016, "process": {"exists": false, "pid": 50016}, "restart_count": 79, "restart_count_per_hour": 6, "rtsp_timeout_count": 144, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 80}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.618, "sample_count": 2642, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 587883, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:31:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.018, "last_frame_pushed_at": "2026-07-02T08:31:50+09:00", "last_frame_read_age_sec": 0.125, "last_frame_read_at": "2026-07-02T08:31:50+09:00", "mjpeg": {"age_sec": 0.122, "exists": true, "mtime": "2026-07-02T08:31:50.325286+09:00", "mtime_epoch": 1782948710.3252857, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 257225}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T08:31:50+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1144.465, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 162, "pushed_frames": 162, "read_frames": 162, "select_timeouts": 12, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.964, "pushed": 8.964, "read": 8.964}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.034, "count": 162, "max_ms": 9.316, "p95_ms": 7.949}, "gstreamer_push": {"avg_ms": 1.577, "count": 162, "max_ms": 7.896, "p95_ms": 3.696}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.009, "count": 162, "max_ms": 79.384, "p95_ms": 64.595}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.066, "count": 162, "max_ms": 92.863, "p95_ms": 90.114}, "yolo11_640_fire_11class": {"avg_ms": 72.914, "count": 162, "max_ms": 84.892, "p95_ms": 82.949}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.129, "count": 162, "max_ms": 85.451, "p95_ms": 84.167}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.158, "count": 162, "max_ms": 108.394, "p95_ms": 105.481}, "read_interval": {"avg_ms": 243.188, "count": 162, "max_ms": 21449.95, "p95_ms": 122.232}, "triton_wait": {"avg_ms": 81.083, "count": 162, "max_ms": 92.877, "p95_ms": 90.127}, "visualize": {"avg_ms": 3.786, "count": 162, "max_ms": 5.937, "p95_ms": 4.479}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.072, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 4899}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9465.766, "limit_mb": 8796093022207.996}, "detail": {"pid": 50382, "restart_count": 80, "start_count": 81}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 80, "h264_decode_error_count": 80, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:31:50+09:00", "last_error_summary": "[h264 @ 0x556d52f2d480] error while decoding MB 112 64, bytestream -10 [in#0/rtsp @ 0x556d52e15e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x556d52e15e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T08:31:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T08:31:52+09:00", "pid": 50382, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.324, "name": "ffmpeg", "pid": 50382, "rss_mb": 58.324, "state": "S (sleeping)", "threads": 1}, "restart_count": 80, "restart_count_per_hour": 7, "rtsp_timeout_count": 144, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 81}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.637, "sample_count": 2604, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 587883, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:31:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.395, "last_frame_pushed_at": "2026-07-02T08:31:50+09:00", "last_frame_read_age_sec": 4.502, "last_frame_read_at": "2026-07-02T08:31:50+09:00", "mjpeg": {"age_sec": 4.379, "exists": true, "mtime": "2026-07-02T08:31:50.445287+09:00", "mtime_epoch": 1782948710.4452875, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 257225}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T08:31:54+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1144.465, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 162, "pushed_frames": 162, "read_frames": 162, "select_timeouts": 8, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.216, "pushed": 7.216, "read": 7.216}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.034, "count": 162, "max_ms": 9.316, "p95_ms": 7.949}, "gstreamer_push": {"avg_ms": 1.577, "count": 162, "max_ms": 7.896, "p95_ms": 3.696}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.009, "count": 162, "max_ms": 79.384, "p95_ms": 64.595}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.066, "count": 162, "max_ms": 92.863, "p95_ms": 90.114}, "yolo11_640_fire_11class": {"avg_ms": 72.914, "count": 162, "max_ms": 84.892, "p95_ms": 82.949}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.129, "count": 162, "max_ms": 85.451, "p95_ms": 84.167}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.158, "count": 162, "max_ms": 108.394, "p95_ms": 105.481}, "read_interval": {"avg_ms": 243.188, "count": 162, "max_ms": 21449.95, "p95_ms": 122.232}, "triton_wait": {"avg_ms": 81.083, "count": 162, "max_ms": 92.877, "p95_ms": 90.127}, "visualize": {"avg_ms": 3.786, "count": 162, "max_ms": 5.937, "p95_ms": 4.479}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.449, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3509}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9514.016, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 7, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x561ae6b0ba80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561ae6b0ba80] error while decoding MB 0 31, bytestream 4377 [h264 @ 0x561ae5e95f80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561ae5e95f80] error while decoding MB 0 36, bytestream 5399 [h264 @ 0x561ae5e41400] left block unavailable for requested intra mode [h264 @ 0x561ae5e41400] error while decoding MB 0 18, bytestream 1876 [h264 @ 0x561ae5e41400] left block unavailable for requested i", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 282, "h264_decode_error_count": 526, "h264_decode_error_count_per_hour": 49, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:35:11+09:00", "last_error_summary": "[h264 @ 0x561ae6b0ba80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561ae6b0ba80] error while decoding MB 0 31, bytestream 4377 [h264 @ 0x561ae5e95f80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561ae5e95f80] error while decoding MB 0 36, bytestream 5399 [h264 @ 0x561ae5e41400] left block unavailable for requested intra mode [h264 @ 0x561ae5e41400] error while decoding MB 0 18, bytestream 1876 [h264 @ 0x561ae5e41400] left block unavailable for requested i", "last_exit_at": "2026-07-02T08:31:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 238.478, "last_started_at": "2026-07-02T08:31:13+09:00", "pid": 50275, "process": {"exists": false, "pid": 50275}, "restart_count": 315, "restart_count_per_hour": 11, "rtsp_timeout_count": 124, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 316}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 129.131, "sample_count": 2323, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 421131, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:35:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.064, "last_frame_pushed_at": "2026-07-02T08:35:11+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-02T08:35:11+09:00", "mjpeg": {"age_sec": 0.048, "exists": true, "mtime": "2026-07-02T08:35:11.677816+09:00", "mtime_epoch": 1782948911.677816, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 213170}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:35:11+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5558.551, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 95, "pushed_frames": 95, "read_frames": 95, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.176, "pushed": 3.176, "read": 3.176}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.063, "count": 95, "max_ms": 17.89, "p95_ms": 16.998}, "gstreamer_push": {"avg_ms": 1.27, "count": 95, "max_ms": 1.456, "p95_ms": 1.344}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.165, "count": 95, "max_ms": 71.608, "p95_ms": 66.596}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.982, "count": 95, "max_ms": 92.724, "p95_ms": 88.804}, "yolo11_640_fire_11class": {"avg_ms": 63.628, "count": 95, "max_ms": 77.144, "p95_ms": 75.989}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.266, "count": 95, "max_ms": 83.101, "p95_ms": 80.771}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.441, "count": 95, "max_ms": 114.808, "p95_ms": 110.972}, "read_interval": {"avg_ms": 315.464, "count": 95, "max_ms": 10128.004, "p95_ms": 127.935}, "triton_wait": {"avg_ms": 74.994, "count": 95, "max_ms": 92.736, "p95_ms": 88.816}, "visualize": {"avg_ms": 3.548, "count": 95, "max_ms": 4.398, "p95_ms": 3.86}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.912, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3509}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9513.68, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T08:35:11+09:00", "last_error_summary": "[h264 @ 0x561ae6b0ba80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561ae6b0ba80] error while decoding MB 0 31, bytestream 4377 [h264 @ 0x561ae5e95f80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561ae5e95f80] error while decoding MB 0 36, bytestream 5399 [h264 @ 0x561ae5e41400] left block unavailable for requested intra mode [h264 @ 0x561ae5e41400] error while decoding MB 0 18, bytestream 1876 [h264 @ 0x561ae5e41400] left block unavailable for requested i", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 282, "h264_decode_error_count": 526, "h264_decode_error_count_per_hour": 49, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:35:11+09:00", "last_error_summary": "[h264 @ 0x561ae6b0ba80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561ae6b0ba80] error while decoding MB 0 31, bytestream 4377 [h264 @ 0x561ae5e95f80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561ae5e95f80] error while decoding MB 0 36, bytestream 5399 [h264 @ 0x561ae5e41400] left block unavailable for requested intra mode [h264 @ 0x561ae5e41400] error while decoding MB 0 18, bytestream 1876 [h264 @ 0x561ae5e41400] left block unavailable for requested i", "last_exit_at": "2026-07-02T08:35:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 238.513, "last_started_at": "2026-07-02T08:31:13+09:00", "pid": 50275, "process": {"exists": false, "pid": 50275}, "restart_count": 315, "restart_count_per_hour": 11, "rtsp_timeout_count": 124, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 316}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 128.75, "sample_count": 2323, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 421132, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:35:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T08:35:11+09:00", "last_frame_read_age_sec": 0.088, "last_frame_read_at": "2026-07-02T08:35:11+09:00", "mjpeg": {"age_sec": 0.082, "exists": true, "mtime": "2026-07-02T08:35:11.677816+09:00", "mtime_epoch": 1782948911.677816, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 213170}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:35:11+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5558.551, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 95, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.206, "pushed": 3.206, "read": 3.172}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.061, "count": 96, "max_ms": 17.89, "p95_ms": 16.998}, "gstreamer_push": {"avg_ms": 1.271, "count": 96, "max_ms": 1.456, "p95_ms": 1.344}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.026, "count": 96, "max_ms": 71.608, "p95_ms": 66.596}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.809, "count": 96, "max_ms": 92.724, "p95_ms": 88.804}, "yolo11_640_fire_11class": {"avg_ms": 63.477, "count": 96, "max_ms": 77.144, "p95_ms": 75.989}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.11, "count": 96, "max_ms": 83.101, "p95_ms": 80.771}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.262, "count": 96, "max_ms": 114.808, "p95_ms": 110.972}, "read_interval": {"avg_ms": 315.464, "count": 95, "max_ms": 10128.004, "p95_ms": 127.935}, "triton_wait": {"avg_ms": 74.821, "count": 96, "max_ms": 92.736, "p95_ms": 88.816}, "visualize": {"avg_ms": 3.547, "count": 96, "max_ms": 4.398, "p95_ms": 3.86}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.946, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3509}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9513.582, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 282, "h264_decode_error_count": 526, "h264_decode_error_count_per_hour": 49, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:35:11+09:00", "last_error_summary": "[h264 @ 0x561ae6b0ba80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561ae6b0ba80] error while decoding MB 0 31, bytestream 4377 [h264 @ 0x561ae5e95f80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561ae5e95f80] error while decoding MB 0 36, bytestream 5399 [h264 @ 0x561ae5e41400] left block unavailable for requested intra mode [h264 @ 0x561ae5e41400] error while decoding MB 0 18, bytestream 1876 [h264 @ 0x561ae5e41400] left block unavailable for requested i", "last_exit_at": "2026-07-02T08:35:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 238.52, "last_started_at": "2026-07-02T08:31:13+09:00", "pid": 50275, "process": {"exists": false, "pid": 50275}, "restart_count": 315, "restart_count_per_hour": 11, "rtsp_timeout_count": 124, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 316}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 128.75, "sample_count": 2323, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 421132, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:35:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-02T08:35:11+09:00", "last_frame_read_age_sec": 0.094, "last_frame_read_at": "2026-07-02T08:35:11+09:00", "mjpeg": {"age_sec": 0.089, "exists": true, "mtime": "2026-07-02T08:35:11.677816+09:00", "mtime_epoch": 1782948911.677816, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 213170}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:35:11+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5558.551, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 96, "pushed_frames": 96, "read_frames": 95, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.205, "pushed": 3.205, "read": 3.172}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.061, "count": 96, "max_ms": 17.89, "p95_ms": 16.998}, "gstreamer_push": {"avg_ms": 1.271, "count": 96, "max_ms": 1.456, "p95_ms": 1.344}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.026, "count": 96, "max_ms": 71.608, "p95_ms": 66.596}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.809, "count": 96, "max_ms": 92.724, "p95_ms": 88.804}, "yolo11_640_fire_11class": {"avg_ms": 63.477, "count": 96, "max_ms": 77.144, "p95_ms": 75.989}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.11, "count": 96, "max_ms": 83.101, "p95_ms": 80.771}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.262, "count": 96, "max_ms": 114.808, "p95_ms": 110.972}, "read_interval": {"avg_ms": 315.464, "count": 95, "max_ms": 10128.004, "p95_ms": 127.935}, "triton_wait": {"avg_ms": 74.821, "count": 96, "max_ms": 92.736, "p95_ms": 88.816}, "visualize": {"avg_ms": 3.547, "count": 96, "max_ms": 4.398, "p95_ms": 3.86}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.953, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3509}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9647.426, "limit_mb": 8796093022207.996}, "detail": {"pid": 50534, "restart_count": 316, "start_count": 317}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 282, "h264_decode_error_count": 526, "h264_decode_error_count_per_hour": 49, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:35:11+09:00", "last_error_summary": "[h264 @ 0x561ae6b0ba80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561ae6b0ba80] error while decoding MB 0 31, bytestream 4377 [h264 @ 0x561ae5e95f80] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561ae5e95f80] error while decoding MB 0 36, bytestream 5399 [h264 @ 0x561ae5e41400] left block unavailable for requested intra mode [h264 @ 0x561ae5e41400] error while decoding MB 0 18, bytestream 1876 [h264 @ 0x561ae5e41400] left block unavailable for requested i", "last_exit_at": "2026-07-02T08:35:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T08:35:14+09:00", "pid": 50534, "process": {"exists": true, "fd_count": 5, "hwm_mb": 144.496, "name": "ffmpeg", "pid": 50534, "rss_mb": 144.496, "state": "S (sleeping)", "threads": 106}, "restart_count": 316, "restart_count_per_hour": 12, "rtsp_timeout_count": 124, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 317}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 125.769, "sample_count": 2292, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 421132, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:35:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.476, "last_frame_pushed_at": "2026-07-02T08:35:11+09:00", "last_frame_read_age_sec": 4.558, "last_frame_read_at": "2026-07-02T08:35:11+09:00", "mjpeg": {"age_sec": 4.461, "exists": true, "mtime": "2026-07-02T08:35:11.769819+09:00", "mtime_epoch": 1782948911.7698185, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 212893}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:35:16+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5558.551, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 60, "pushed_frames": 60, "read_frames": 59, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.001, "pushed": 2.001, "read": 1.967}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.131, "count": 60, "max_ms": 17.89, "p95_ms": 17.016}, "gstreamer_push": {"avg_ms": 1.268, "count": 60, "max_ms": 1.456, "p95_ms": 1.344}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.405, "count": 60, "max_ms": 67.945, "p95_ms": 66.591}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.719, "count": 60, "max_ms": 90.573, "p95_ms": 88.377}, "yolo11_640_fire_11class": {"avg_ms": 58.195, "count": 60, "max_ms": 77.144, "p95_ms": 74.843}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.018, "count": 60, "max_ms": 82.848, "p95_ms": 79.115}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.221, "count": 60, "max_ms": 112.438, "p95_ms": 110.216}, "read_interval": {"avg_ms": 432.889, "count": 59, "max_ms": 10128.004, "p95_ms": 127.935}, "triton_wait": {"avg_ms": 68.731, "count": 60, "max_ms": 90.586, "p95_ms": 88.389}, "visualize": {"avg_ms": 3.569, "count": 60, "max_ms": 4.398, "p95_ms": 4.072}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.988, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 4911}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9321.246, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5623cc416dc0] error while decoding MB 100 38, bytestream -33 [in#0/rtsp @ 0x5623cc319e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5623cc319e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 81, "h264_decode_error_count": 81, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:35:16+09:00", "last_error_summary": "[h264 @ 0x5623cc416dc0] error while decoding MB 100 38, bytestream -33 [in#0/rtsp @ 0x5623cc319e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5623cc319e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T08:31:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 203.803, "last_started_at": "2026-07-02T08:31:52+09:00", "pid": 50382, "process": {"exists": false, "pid": 50382}, "restart_count": 80, "restart_count_per_hour": 6, "rtsp_timeout_count": 146, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 81}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.59, "sample_count": 2161, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 589371, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:35:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-02T08:35:16+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-02T08:35:16+09:00", "mjpeg": {"age_sec": 0.06, "exists": true, "mtime": "2026-07-02T08:35:16.565956+09:00", "mtime_epoch": 1782948916.5659556, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 254904}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T08:35:16+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1147.875, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 36, "pushed_frames": 36, "read_frames": 36, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.204, "pushed": 1.204, "read": 1.204}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.723, "count": 36, "max_ms": 6.431, "p95_ms": 5.845}, "gstreamer_push": {"avg_ms": 1.55, "count": 36, "max_ms": 3.879, "p95_ms": 3.764}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.894, "count": 36, "max_ms": 75.17, "p95_ms": 69.635}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.269, "count": 36, "max_ms": 94.371, "p95_ms": 94.177}, "yolo11_640_fire_11class": {"avg_ms": 68.284, "count": 36, "max_ms": 84.359, "p95_ms": 83.926}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.438, "count": 36, "max_ms": 85.102, "p95_ms": 85.009}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.81, "count": 36, "max_ms": 110.133, "p95_ms": 108.629}, "read_interval": {"avg_ms": 831.85, "count": 36, "max_ms": 16042.123, "p95_ms": 10167.089}, "triton_wait": {"avg_ms": 79.282, "count": 36, "max_ms": 94.384, "p95_ms": 94.19}, "visualize": {"avg_ms": 3.663, "count": 36, "max_ms": 4.147, "p95_ms": 3.902}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.905, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 4911}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9320.504, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T08:35:16+09:00", "last_error_summary": "[h264 @ 0x5623cc416dc0] error while decoding MB 100 38, bytestream -33 [in#0/rtsp @ 0x5623cc319e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5623cc319e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 81, "h264_decode_error_count": 81, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:35:16+09:00", "last_error_summary": "[h264 @ 0x5623cc416dc0] error while decoding MB 100 38, bytestream -33 [in#0/rtsp @ 0x5623cc319e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5623cc319e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T08:35:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 203.856, "last_started_at": "2026-07-02T08:31:52+09:00", "pid": 50382, "process": {"exists": false, "pid": 50382}, "restart_count": 80, "restart_count_per_hour": 6, "rtsp_timeout_count": 146, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 81}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.607, "sample_count": 2162, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 589372, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:35:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T08:35:16+09:00", "last_frame_read_age_sec": 0.118, "last_frame_read_at": "2026-07-02T08:35:16+09:00", "mjpeg": {"age_sec": 0.114, "exists": true, "mtime": "2026-07-02T08:35:16.565956+09:00", "mtime_epoch": 1782948916.5659556, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 254904}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T08:35:16+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1147.875, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 37, "pushed_frames": 37, "read_frames": 36, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.235, "pushed": 1.235, "read": 1.202}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.722, "count": 37, "max_ms": 6.431, "p95_ms": 5.845}, "gstreamer_push": {"avg_ms": 1.543, "count": 37, "max_ms": 3.879, "p95_ms": 3.764}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.91, "count": 37, "max_ms": 75.17, "p95_ms": 69.635}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.726, "count": 37, "max_ms": 96.18, "p95_ms": 94.371}, "yolo11_640_fire_11class": {"avg_ms": 68.41, "count": 37, "max_ms": 84.359, "p95_ms": 83.926}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.753, "count": 37, "max_ms": 85.102, "p95_ms": 85.009}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.263, "count": 37, "max_ms": 110.58, "p95_ms": 110.133}, "read_interval": {"avg_ms": 831.85, "count": 36, "max_ms": 16042.123, "p95_ms": 10167.089}, "triton_wait": {"avg_ms": 79.739, "count": 37, "max_ms": 96.192, "p95_ms": 94.384}, "visualize": {"avg_ms": 3.662, "count": 37, "max_ms": 4.147, "p95_ms": 3.902}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.958, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 4911}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9323.082, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 81, "h264_decode_error_count": 81, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:35:16+09:00", "last_error_summary": "[h264 @ 0x5623cc416dc0] error while decoding MB 100 38, bytestream -33 [in#0/rtsp @ 0x5623cc319e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5623cc319e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T08:35:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 203.863, "last_started_at": "2026-07-02T08:31:52+09:00", "pid": 50382, "process": {"exists": false, "pid": 50382}, "restart_count": 80, "restart_count_per_hour": 6, "rtsp_timeout_count": 146, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 81}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.607, "sample_count": 2162, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 589372, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:35:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-02T08:35:16+09:00", "last_frame_read_age_sec": 0.125, "last_frame_read_at": "2026-07-02T08:35:16+09:00", "mjpeg": {"age_sec": 0.121, "exists": true, "mtime": "2026-07-02T08:35:16.565956+09:00", "mtime_epoch": 1782948916.5659556, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 254904}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T08:35:16+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1147.875, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 37, "pushed_frames": 37, "read_frames": 36, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.235, "pushed": 1.235, "read": 1.201}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.722, "count": 37, "max_ms": 6.431, "p95_ms": 5.845}, "gstreamer_push": {"avg_ms": 1.543, "count": 37, "max_ms": 3.879, "p95_ms": 3.764}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 45.91, "count": 37, "max_ms": 75.17, "p95_ms": 69.635}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.726, "count": 37, "max_ms": 96.18, "p95_ms": 94.371}, "yolo11_640_fire_11class": {"avg_ms": 68.41, "count": 37, "max_ms": 84.359, "p95_ms": 83.926}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.753, "count": 37, "max_ms": 85.102, "p95_ms": 85.009}}, "truncated_count": 0}, "processing_total": {"avg_ms": 94.263, "count": 37, "max_ms": 110.58, "p95_ms": 110.133}, "read_interval": {"avg_ms": 831.85, "count": 36, "max_ms": 16042.123, "p95_ms": 10167.089}, "triton_wait": {"avg_ms": 79.739, "count": 37, "max_ms": 96.192, "p95_ms": 94.384}, "visualize": {"avg_ms": 3.662, "count": 37, "max_ms": 4.147, "p95_ms": 3.902}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.965, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 4911}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9524.984, "limit_mb": 8796093022207.996}, "detail": {"pid": 50641, "restart_count": 81, "start_count": 82}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 81, "h264_decode_error_count": 81, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:35:16+09:00", "last_error_summary": "[h264 @ 0x5623cc416dc0] error while decoding MB 100 38, bytestream -33 [in#0/rtsp @ 0x5623cc319e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5623cc319e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T08:35:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T08:35:19+09:00", "pid": 50641, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.195, "name": "ffmpeg", "pid": 50641, "rss_mb": 136.195, "state": "S (sleeping)", "threads": 19}, "restart_count": 81, "restart_count_per_hour": 7, "rtsp_timeout_count": 146, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 82}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.626, "sample_count": 2117, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 589372, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:35:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.356, "last_frame_pushed_at": "2026-07-02T08:35:16+09:00", "last_frame_read_age_sec": 4.468, "last_frame_read_at": "2026-07-02T08:35:16+09:00", "mjpeg": {"age_sec": 4.332, "exists": true, "mtime": "2026-07-02T08:35:16.697959+09:00", "mtime_epoch": 1782948916.6979594, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 254892}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T08:35:21+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1147.875, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.835, "pushed": 0.835, "read": 0.835}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.781, "count": 13, "max_ms": 6.431, "p95_ms": 6.431}, "gstreamer_push": {"avg_ms": 1.354, "count": 13, "max_ms": 1.41, "p95_ms": 1.41}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 42.083, "count": 13, "max_ms": 60.701, "p95_ms": 60.701}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.215, "count": 13, "max_ms": 96.18, "p95_ms": 96.18}, "yolo11_640_fire_11class": {"avg_ms": 51.017, "count": 13, "max_ms": 72.95, "p95_ms": 72.95}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.057, "count": 13, "max_ms": 82.088, "p95_ms": 82.088}}, "truncated_count": 0}, "processing_total": {"avg_ms": 73.924, "count": 13, "max_ms": 110.58, "p95_ms": 110.58}, "read_interval": {"avg_ms": 2088.44, "count": 13, "max_ms": 16042.123, "p95_ms": 16042.123}, "triton_wait": {"avg_ms": 59.227, "count": 13, "max_ms": 96.192, "p95_ms": 96.192}, "visualize": {"avg_ms": 3.698, "count": 13, "max_ms": 3.902, "p95_ms": 3.902}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 15.576, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3534}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9513.277, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55d5e1b52380] error while decoding MB 77 18, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 283, "h264_decode_error_count": 527, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:41:09+09:00", "last_error_summary": "[h264 @ 0x55d5e1b52380] error while decoding MB 77 18, bytestream -5", "last_exit_at": "2026-07-02T08:35:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 355.259, "last_started_at": "2026-07-02T08:35:14+09:00", "pid": 50534, "process": {"exists": false, "pid": 50534}, "restart_count": 316, "restart_count_per_hour": 11, "rtsp_timeout_count": 124, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 317}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.997, "sample_count": 2577, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 424176, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:41:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.068, "last_frame_pushed_at": "2026-07-02T08:41:09+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-02T08:41:09+09:00", "mjpeg": {"age_sec": 0.05, "exists": true, "mtime": "2026-07-02T08:41:09.438925+09:00", "mtime_epoch": 1782949269.4389246, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188637}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:41:09+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5560.098, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 255, "pushed_frames": 255, "read_frames": 255, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.523, "pushed": 8.523, "read": 8.523}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.076, "count": 255, "max_ms": 15.648, "p95_ms": 14.775}, "gstreamer_push": {"avg_ms": 1.257, "count": 255, "max_ms": 1.418, "p95_ms": 1.334}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.416, "count": 255, "max_ms": 74.331, "p95_ms": 67.015}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.146, "count": 255, "max_ms": 88.732, "p95_ms": 84.248}, "yolo11_640_fire_11class": {"avg_ms": 71.542, "count": 255, "max_ms": 78.312, "p95_ms": 76.249}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.715, "count": 255, "max_ms": 84.501, "p95_ms": 77.57}}, "truncated_count": 0}, "processing_total": {"avg_ms": 101.698, "count": 255, "max_ms": 111.472, "p95_ms": 106.959}, "read_interval": {"avg_ms": 117.529, "count": 255, "max_ms": 127.791, "p95_ms": 123.09}, "triton_wait": {"avg_ms": 79.158, "count": 255, "max_ms": 88.746, "p95_ms": 84.26}, "visualize": {"avg_ms": 3.554, "count": 255, "max_ms": 4.468, "p95_ms": 3.865}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.919, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3534}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9512.418, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T08:41:09+09:00", "last_error_summary": "[h264 @ 0x55d5e1b52380] error while decoding MB 77 18, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 283, "h264_decode_error_count": 527, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:41:09+09:00", "last_error_summary": "[h264 @ 0x55d5e1b52380] error while decoding MB 77 18, bytestream -5", "last_exit_at": "2026-07-02T08:41:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 355.318, "last_started_at": "2026-07-02T08:35:14+09:00", "pid": 50534, "process": {"exists": false, "pid": 50534}, "restart_count": 316, "restart_count_per_hour": 11, "rtsp_timeout_count": 124, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 317}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.997, "sample_count": 2578, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 424177, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:41:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T08:41:09+09:00", "last_frame_read_age_sec": 0.114, "last_frame_read_at": "2026-07-02T08:41:09+09:00", "mjpeg": {"age_sec": 0.109, "exists": true, "mtime": "2026-07-02T08:41:09.438925+09:00", "mtime_epoch": 1782949269.4389246, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188637}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:41:09+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5560.098, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 256, "pushed_frames": 256, "read_frames": 255, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.54, "pushed": 8.54, "read": 8.506}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.076, "count": 256, "max_ms": 15.648, "p95_ms": 14.775}, "gstreamer_push": {"avg_ms": 1.257, "count": 256, "max_ms": 1.418, "p95_ms": 1.336}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.428, "count": 256, "max_ms": 74.331, "p95_ms": 67.015}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.157, "count": 256, "max_ms": 88.732, "p95_ms": 84.248}, "yolo11_640_fire_11class": {"avg_ms": 71.557, "count": 256, "max_ms": 78.312, "p95_ms": 76.249}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.736, "count": 256, "max_ms": 84.501, "p95_ms": 77.57}}, "truncated_count": 0}, "processing_total": {"avg_ms": 101.708, "count": 256, "max_ms": 111.472, "p95_ms": 106.959}, "read_interval": {"avg_ms": 117.529, "count": 255, "max_ms": 127.791, "p95_ms": 123.09}, "triton_wait": {"avg_ms": 79.169, "count": 256, "max_ms": 88.746, "p95_ms": 84.26}, "visualize": {"avg_ms": 3.555, "count": 256, "max_ms": 4.468, "p95_ms": 3.865}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.977, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3534}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9512.344, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 283, "h264_decode_error_count": 527, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:41:09+09:00", "last_error_summary": "[h264 @ 0x55d5e1b52380] error while decoding MB 77 18, bytestream -5", "last_exit_at": "2026-07-02T08:41:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 355.327, "last_started_at": "2026-07-02T08:35:14+09:00", "pid": 50534, "process": {"exists": false, "pid": 50534}, "restart_count": 316, "restart_count_per_hour": 11, "rtsp_timeout_count": 124, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 317}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 112.997, "sample_count": 2578, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 424177, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:41:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-02T08:41:09+09:00", "last_frame_read_age_sec": 0.122, "last_frame_read_at": "2026-07-02T08:41:09+09:00", "mjpeg": {"age_sec": 0.117, "exists": true, "mtime": "2026-07-02T08:41:09.438925+09:00", "mtime_epoch": 1782949269.4389246, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188637}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:41:09+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5560.098, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 256, "pushed_frames": 256, "read_frames": 255, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.537, "pushed": 8.537, "read": 8.504}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.076, "count": 256, "max_ms": 15.648, "p95_ms": 14.775}, "gstreamer_push": {"avg_ms": 1.257, "count": 256, "max_ms": 1.418, "p95_ms": 1.336}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.428, "count": 256, "max_ms": 74.331, "p95_ms": 67.015}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.157, "count": 256, "max_ms": 88.732, "p95_ms": 84.248}, "yolo11_640_fire_11class": {"avg_ms": 71.557, "count": 256, "max_ms": 78.312, "p95_ms": 76.249}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.736, "count": 256, "max_ms": 84.501, "p95_ms": 77.57}}, "truncated_count": 0}, "processing_total": {"avg_ms": 101.708, "count": 256, "max_ms": 111.472, "p95_ms": 106.959}, "read_interval": {"avg_ms": 117.529, "count": 255, "max_ms": 127.791, "p95_ms": 123.09}, "triton_wait": {"avg_ms": 79.169, "count": 256, "max_ms": 88.746, "p95_ms": 84.26}, "visualize": {"avg_ms": 3.555, "count": 256, "max_ms": 4.468, "p95_ms": 3.865}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.986, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3534}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9617.324, "limit_mb": 8796093022207.996}, "detail": {"pid": 50793, "restart_count": 317, "start_count": 318}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 283, "h264_decode_error_count": 527, "h264_decode_error_count_per_hour": 41, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:41:09+09:00", "last_error_summary": "[h264 @ 0x55d5e1b52380] error while decoding MB 77 18, bytestream -5", "last_exit_at": "2026-07-02T08:41:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T08:41:11+09:00", "pid": 50793, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.234, "name": "ffmpeg", "pid": 50793, "rss_mb": 139.234, "state": "S (sleeping)", "threads": 106}, "restart_count": 317, "restart_count_per_hour": 12, "rtsp_timeout_count": 124, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 318}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 113.016, "sample_count": 2540, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 424177, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:41:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.331, "last_frame_pushed_at": "2026-07-02T08:41:09+09:00", "last_frame_read_age_sec": 4.438, "last_frame_read_at": "2026-07-02T08:41:09+09:00", "mjpeg": {"age_sec": 4.313, "exists": true, "mtime": "2026-07-02T08:41:09.558927+09:00", "mtime_epoch": 1782949269.5589273, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188972}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:41:13+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5560.098, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 219, "pushed_frames": 219, "read_frames": 218, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.321, "pushed": 7.321, "read": 7.288}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.072, "count": 219, "max_ms": 15.648, "p95_ms": 14.726}, "gstreamer_push": {"avg_ms": 1.262, "count": 219, "max_ms": 1.418, "p95_ms": 1.342}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.308, "count": 219, "max_ms": 74.331, "p95_ms": 67.019}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.98, "count": 219, "max_ms": 88.732, "p95_ms": 84.278}, "yolo11_640_fire_11class": {"avg_ms": 71.294, "count": 219, "max_ms": 78.312, "p95_ms": 76.059}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.447, "count": 219, "max_ms": 84.501, "p95_ms": 77.488}}, "truncated_count": 0}, "processing_total": {"avg_ms": 101.533, "count": 219, "max_ms": 111.472, "p95_ms": 107.386}, "read_interval": {"avg_ms": 117.337, "count": 218, "max_ms": 127.791, "p95_ms": 123.212}, "triton_wait": {"avg_ms": 78.992, "count": 219, "max_ms": 88.746, "p95_ms": 84.29}, "visualize": {"avg_ms": 3.556, "count": 219, "max_ms": 4.468, "p95_ms": 3.853}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.912, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3551}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9515.566, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 8, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x556dc62be180] left block unavailable for requested intra mode [h264 @ 0x556dc62be180] error while decoding MB 0 21, bytestream 32772 [h264 @ 0x556dc6cbb880] left block unavailable for requested intra mode [h264 @ 0x556dc6cbb880] error while decoding MB 0 17, bytestream 931 [h264 @ 0x556dc618cb00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556dc618cb00] error while decoding MB 0 29, bytestream 5047 [h264 @ 0x556dc6b1db80] top block unavailable for requested intra mo", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 284, "h264_decode_error_count": 535, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:45:36+09:00", "last_error_summary": "[h264 @ 0x556dc62be180] left block unavailable for requested intra mode [h264 @ 0x556dc62be180] error while decoding MB 0 21, bytestream 32772 [h264 @ 0x556dc6cbb880] left block unavailable for requested intra mode [h264 @ 0x556dc6cbb880] error while decoding MB 0 17, bytestream 931 [h264 @ 0x556dc618cb00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556dc618cb00] error while decoding MB 0 29, bytestream 5047 [h264 @ 0x556dc6b1db80] top block unavailable for requested intra mo", "last_exit_at": "2026-07-02T08:41:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 264.478, "last_started_at": "2026-07-02T08:41:11+09:00", "pid": 50793, "process": {"exists": false, "pid": 50793}, "restart_count": 317, "restart_count_per_hour": 11, "rtsp_timeout_count": 126, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 318}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.939, "sample_count": 2305, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 426199, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:45:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.08, "last_frame_pushed_at": "2026-07-02T08:45:36+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-02T08:45:36+09:00", "mjpeg": {"age_sec": 0.064, "exists": true, "mtime": "2026-07-02T08:45:36.284727+09:00", "mtime_epoch": 1782949536.2847273, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184514}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:45:36+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5561.129, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 93, "pushed_frames": 93, "read_frames": 93, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.107, "pushed": 3.107, "read": 3.107}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 13.92, "count": 93, "max_ms": 15.6, "p95_ms": 14.344}, "gstreamer_push": {"avg_ms": 1.261, "count": 93, "max_ms": 1.36, "p95_ms": 1.346}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.588, "count": 93, "max_ms": 77.992, "p95_ms": 67.596}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.726, "count": 93, "max_ms": 92.525, "p95_ms": 85.562}, "yolo11_640_fire_11class": {"avg_ms": 71.238, "count": 93, "max_ms": 78.365, "p95_ms": 77.139}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.865, "count": 93, "max_ms": 81.455, "p95_ms": 78.947}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.205, "count": 93, "max_ms": 114.418, "p95_ms": 107.789}, "read_interval": {"avg_ms": 322.256, "count": 93, "max_ms": 19079.961, "p95_ms": 125.319}, "triton_wait": {"avg_ms": 80.737, "count": 93, "max_ms": 92.537, "p95_ms": 85.573}, "visualize": {"avg_ms": 3.63, "count": 93, "max_ms": 4.924, "p95_ms": 3.907}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.932, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3551}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9515.031, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T08:45:36+09:00", "last_error_summary": "[h264 @ 0x556dc62be180] left block unavailable for requested intra mode [h264 @ 0x556dc62be180] error while decoding MB 0 21, bytestream 32772 [h264 @ 0x556dc6cbb880] left block unavailable for requested intra mode [h264 @ 0x556dc6cbb880] error while decoding MB 0 17, bytestream 931 [h264 @ 0x556dc618cb00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556dc618cb00] error while decoding MB 0 29, bytestream 5047 [h264 @ 0x556dc6b1db80] top block unavailable for requested intra mo", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 284, "h264_decode_error_count": 535, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:45:36+09:00", "last_error_summary": "[h264 @ 0x556dc62be180] left block unavailable for requested intra mode [h264 @ 0x556dc62be180] error while decoding MB 0 21, bytestream 32772 [h264 @ 0x556dc6cbb880] left block unavailable for requested intra mode [h264 @ 0x556dc6cbb880] error while decoding MB 0 17, bytestream 931 [h264 @ 0x556dc618cb00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556dc618cb00] error while decoding MB 0 29, bytestream 5047 [h264 @ 0x556dc6b1db80] top block unavailable for requested intra mo", "last_exit_at": "2026-07-02T08:45:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 264.534, "last_started_at": "2026-07-02T08:41:11+09:00", "pid": 50793, "process": {"exists": false, "pid": 50793}, "restart_count": 317, "restart_count_per_hour": 11, "rtsp_timeout_count": 126, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 318}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.939, "sample_count": 2305, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 426200, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:45:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T08:45:36+09:00", "last_frame_read_age_sec": 0.121, "last_frame_read_at": "2026-07-02T08:45:36+09:00", "mjpeg": {"age_sec": 0.12, "exists": true, "mtime": "2026-07-02T08:45:36.284727+09:00", "mtime_epoch": 1782949536.2847273, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184514}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:45:36+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5561.129, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 94, "pushed_frames": 94, "read_frames": 93, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.135, "pushed": 3.135, "read": 3.101}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 13.923, "count": 94, "max_ms": 15.6, "p95_ms": 14.344}, "gstreamer_push": {"avg_ms": 1.261, "count": 94, "max_ms": 1.36, "p95_ms": 1.346}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.683, "count": 94, "max_ms": 77.992, "p95_ms": 68.535}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.825, "count": 94, "max_ms": 92.525, "p95_ms": 85.664}, "yolo11_640_fire_11class": {"avg_ms": 71.344, "count": 94, "max_ms": 81.198, "p95_ms": 77.212}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.949, "count": 94, "max_ms": 81.455, "p95_ms": 79.143}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.312, "count": 94, "max_ms": 114.418, "p95_ms": 107.829}, "read_interval": {"avg_ms": 322.256, "count": 93, "max_ms": 19079.961, "p95_ms": 125.319}, "triton_wait": {"avg_ms": 80.836, "count": 94, "max_ms": 92.537, "p95_ms": 85.675}, "visualize": {"avg_ms": 3.63, "count": 94, "max_ms": 4.924, "p95_ms": 3.907}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.988, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3551}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9515.023, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 284, "h264_decode_error_count": 535, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:45:36+09:00", "last_error_summary": "[h264 @ 0x556dc62be180] left block unavailable for requested intra mode [h264 @ 0x556dc62be180] error while decoding MB 0 21, bytestream 32772 [h264 @ 0x556dc6cbb880] left block unavailable for requested intra mode [h264 @ 0x556dc6cbb880] error while decoding MB 0 17, bytestream 931 [h264 @ 0x556dc618cb00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556dc618cb00] error while decoding MB 0 29, bytestream 5047 [h264 @ 0x556dc6b1db80] top block unavailable for requested intra mo", "last_exit_at": "2026-07-02T08:45:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 264.544, "last_started_at": "2026-07-02T08:41:11+09:00", "pid": 50793, "process": {"exists": false, "pid": 50793}, "restart_count": 317, "restart_count_per_hour": 11, "rtsp_timeout_count": 126, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 318}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 116.939, "sample_count": 2305, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 426200, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:45:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-02T08:45:36+09:00", "last_frame_read_age_sec": 0.131, "last_frame_read_at": "2026-07-02T08:45:36+09:00", "mjpeg": {"age_sec": 0.129, "exists": true, "mtime": "2026-07-02T08:45:36.284727+09:00", "mtime_epoch": 1782949536.2847273, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184514}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:45:36+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5561.129, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 94, "pushed_frames": 94, "read_frames": 93, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.134, "pushed": 3.134, "read": 3.1}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 13.928, "count": 93, "max_ms": 15.6, "p95_ms": 14.344}, "gstreamer_push": {"avg_ms": 1.261, "count": 94, "max_ms": 1.36, "p95_ms": 1.346}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.678, "count": 93, "max_ms": 77.992, "p95_ms": 68.535}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.822, "count": 93, "max_ms": 92.525, "p95_ms": 85.664}, "yolo11_640_fire_11class": {"avg_ms": 71.327, "count": 93, "max_ms": 81.198, "p95_ms": 77.212}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.917, "count": 93, "max_ms": 81.455, "p95_ms": 79.143}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.312, "count": 94, "max_ms": 114.418, "p95_ms": 107.829}, "read_interval": {"avg_ms": 322.256, "count": 93, "max_ms": 19079.961, "p95_ms": 125.319}, "triton_wait": {"avg_ms": 80.833, "count": 93, "max_ms": 92.537, "p95_ms": 85.675}, "visualize": {"avg_ms": 3.631, "count": 93, "max_ms": 4.924, "p95_ms": 3.907}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3551}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9619.496, "limit_mb": 8796093022207.996}, "detail": {"pid": 50900, "restart_count": 318, "start_count": 319}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 284, "h264_decode_error_count": 535, "h264_decode_error_count_per_hour": 46, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:45:36+09:00", "last_error_summary": "[h264 @ 0x556dc62be180] left block unavailable for requested intra mode [h264 @ 0x556dc62be180] error while decoding MB 0 21, bytestream 32772 [h264 @ 0x556dc6cbb880] left block unavailable for requested intra mode [h264 @ 0x556dc6cbb880] error while decoding MB 0 17, bytestream 931 [h264 @ 0x556dc618cb00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556dc618cb00] error while decoding MB 0 29, bytestream 5047 [h264 @ 0x556dc6b1db80] top block unavailable for requested intra mo", "last_exit_at": "2026-07-02T08:45:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T08:45:38+09:00", "pid": 50900, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.773, "name": "ffmpeg", "pid": 50900, "rss_mb": 136.773, "state": "S (sleeping)", "threads": 106}, "restart_count": 318, "restart_count_per_hour": 12, "rtsp_timeout_count": 126, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 319}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 118.445, "sample_count": 2269, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 426200, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:45:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.331, "last_frame_pushed_at": "2026-07-02T08:45:36+09:00", "last_frame_read_age_sec": 4.446, "last_frame_read_at": "2026-07-02T08:45:36+09:00", "mjpeg": {"age_sec": 4.317, "exists": true, "mtime": "2026-07-02T08:45:36.412730+09:00", "mtime_epoch": 1782949536.41273, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 184506}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:45:40+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5561.129, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 57, "pushed_frames": 57, "read_frames": 57, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.9, "pushed": 1.9, "read": 1.9}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 13.958, "count": 57, "max_ms": 15.6, "p95_ms": 14.265}, "gstreamer_push": {"avg_ms": 1.272, "count": 57, "max_ms": 1.36, "p95_ms": 1.351}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.685, "count": 57, "max_ms": 77.992, "p95_ms": 67.596}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.064, "count": 57, "max_ms": 92.525, "p95_ms": 86.025}, "yolo11_640_fire_11class": {"avg_ms": 70.112, "count": 57, "max_ms": 81.198, "p95_ms": 75.917}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.797, "count": 57, "max_ms": 80.695, "p95_ms": 78.945}}, "truncated_count": 0}, "processing_total": {"avg_ms": 102.587, "count": 57, "max_ms": 114.418, "p95_ms": 107.829}, "read_interval": {"avg_ms": 450.354, "count": 57, "max_ms": 19079.961, "p95_ms": 125.319}, "triton_wait": {"avg_ms": 80.076, "count": 57, "max_ms": 92.537, "p95_ms": 86.037}, "visualize": {"avg_ms": 3.649, "count": 57, "max_ms": 4.924, "p95_ms": 3.966}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3572}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9520.438, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T08:45:36+09:00", "last_error_summary": "[h264 @ 0x556dc62be180] left block unavailable for requested intra mode [h264 @ 0x556dc62be180] error while decoding MB 0 21, bytestream 32772 [h264 @ 0x556dc6cbb880] left block unavailable for requested intra mode [h264 @ 0x556dc6cbb880] error while decoding MB 0 17, bytestream 931 [h264 @ 0x556dc618cb00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556dc618cb00] error while decoding MB 0 29, bytestream 5047 [h264 @ 0x556dc6b1db80] top block unavailable for requested intra mo", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 284, "h264_decode_error_count": 535, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:45:36+09:00", "last_error_summary": "[h264 @ 0x556dc62be180] left block unavailable for requested intra mode [h264 @ 0x556dc62be180] error while decoding MB 0 21, bytestream 32772 [h264 @ 0x556dc6cbb880] left block unavailable for requested intra mode [h264 @ 0x556dc6cbb880] error while decoding MB 0 17, bytestream 931 [h264 @ 0x556dc618cb00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556dc618cb00] error while decoding MB 0 29, bytestream 5047 [h264 @ 0x556dc6b1db80] top block unavailable for requested intra mo", "last_exit_at": "2026-07-02T08:50:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 299.964, "last_started_at": "2026-07-02T08:45:38+09:00", "pid": 50900, "process": {"exists": false, "pid": 50900}, "restart_count": 318, "restart_count_per_hour": 11, "rtsp_timeout_count": 126, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 319}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.39, "sample_count": 2487, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 428687, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:50:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-02T08:50:38+09:00", "last_frame_read_age_sec": 0.114, "last_frame_read_at": "2026-07-02T08:50:38+09:00", "mjpeg": {"age_sec": 0.11, "exists": true, "mtime": "2026-07-02T08:50:38.582745+09:00", "mtime_epoch": 1782949838.5827448, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 201663}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:50:38+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5562.16, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 250, "pushed_frames": 250, "read_frames": 249, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.34, "pushed": 8.34, "read": 8.307}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.131, "count": 250, "max_ms": 23.574, "p95_ms": 15.14}, "gstreamer_push": {"avg_ms": 1.251, "count": 250, "max_ms": 1.474, "p95_ms": 1.324}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.949, "count": 250, "max_ms": 71.828, "p95_ms": 67.645}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.254, "count": 250, "max_ms": 91.272, "p95_ms": 87.579}, "yolo11_640_fire_11class": {"avg_ms": 72.66, "count": 250, "max_ms": 80.579, "p95_ms": 77.317}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.49, "count": 250, "max_ms": 87.731, "p95_ms": 81.776}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.879, "count": 250, "max_ms": 114.44, "p95_ms": 109.817}, "read_interval": {"avg_ms": 120.345, "count": 249, "max_ms": 131.357, "p95_ms": 125.868}, "triton_wait": {"avg_ms": 82.266, "count": 250, "max_ms": 91.284, "p95_ms": 87.591}, "visualize": {"avg_ms": 3.56, "count": 250, "max_ms": 4.511, "p95_ms": 3.805}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.976, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3572}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9520.496, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 284, "h264_decode_error_count": 535, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:45:36+09:00", "last_error_summary": "[h264 @ 0x556dc62be180] left block unavailable for requested intra mode [h264 @ 0x556dc62be180] error while decoding MB 0 21, bytestream 32772 [h264 @ 0x556dc6cbb880] left block unavailable for requested intra mode [h264 @ 0x556dc6cbb880] error while decoding MB 0 17, bytestream 931 [h264 @ 0x556dc618cb00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556dc618cb00] error while decoding MB 0 29, bytestream 5047 [h264 @ 0x556dc6b1db80] top block unavailable for requested intra mo", "last_exit_at": "2026-07-02T08:50:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 299.972, "last_started_at": "2026-07-02T08:45:38+09:00", "pid": 50900, "process": {"exists": false, "pid": 50900}, "restart_count": 318, "restart_count_per_hour": 11, "rtsp_timeout_count": 126, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 319}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.39, "sample_count": 2487, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 428687, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:50:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-02T08:50:38+09:00", "last_frame_read_age_sec": 0.122, "last_frame_read_at": "2026-07-02T08:50:38+09:00", "mjpeg": {"age_sec": 0.118, "exists": true, "mtime": "2026-07-02T08:50:38.582745+09:00", "mtime_epoch": 1782949838.5827448, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 201663}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:50:38+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5562.16, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 250, "pushed_frames": 250, "read_frames": 249, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.338, "pushed": 8.338, "read": 8.304}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.131, "count": 250, "max_ms": 23.574, "p95_ms": 15.14}, "gstreamer_push": {"avg_ms": 1.251, "count": 250, "max_ms": 1.474, "p95_ms": 1.324}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.949, "count": 250, "max_ms": 71.828, "p95_ms": 67.645}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.254, "count": 250, "max_ms": 91.272, "p95_ms": 87.579}, "yolo11_640_fire_11class": {"avg_ms": 72.66, "count": 250, "max_ms": 80.579, "p95_ms": 77.317}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.49, "count": 250, "max_ms": 87.731, "p95_ms": 81.776}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.879, "count": 250, "max_ms": 114.44, "p95_ms": 109.817}, "read_interval": {"avg_ms": 120.345, "count": 249, "max_ms": 131.357, "p95_ms": 125.868}, "triton_wait": {"avg_ms": 82.266, "count": 250, "max_ms": 91.284, "p95_ms": 87.591}, "visualize": {"avg_ms": 3.56, "count": 250, "max_ms": 4.511, "p95_ms": 3.805}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.984, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3572}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9635.848, "limit_mb": 8796093022207.996}, "detail": {"pid": 51007, "restart_count": 319, "start_count": 320}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 284, "h264_decode_error_count": 535, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:45:36+09:00", "last_error_summary": "[h264 @ 0x556dc62be180] left block unavailable for requested intra mode [h264 @ 0x556dc62be180] error while decoding MB 0 21, bytestream 32772 [h264 @ 0x556dc6cbb880] left block unavailable for requested intra mode [h264 @ 0x556dc6cbb880] error while decoding MB 0 17, bytestream 931 [h264 @ 0x556dc618cb00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556dc618cb00] error while decoding MB 0 29, bytestream 5047 [h264 @ 0x556dc6b1db80] top block unavailable for requested intra mo", "last_exit_at": "2026-07-02T08:50:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T08:50:41+09:00", "pid": 51007, "process": {"exists": true, "fd_count": 5, "hwm_mb": 149.93, "name": "ffmpeg", "pid": 51007, "rss_mb": 149.93, "state": "S (sleeping)", "threads": 106}, "restart_count": 319, "restart_count_per_hour": 12, "rtsp_timeout_count": 126, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 320}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 111.367, "sample_count": 2465, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 428687, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:50:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.645, "last_frame_pushed_at": "2026-07-02T08:50:38+09:00", "last_frame_read_age_sec": 4.752, "last_frame_read_at": "2026-07-02T08:50:38+09:00", "mjpeg": {"age_sec": 4.627, "exists": true, "mtime": "2026-07-02T08:50:38.702747+09:00", "mtime_epoch": 1782949838.702747, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 201717}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:50:43+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5562.16, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 211, "pushed_frames": 211, "read_frames": 210, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.057, "pushed": 7.057, "read": 7.024}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.138, "count": 211, "max_ms": 23.574, "p95_ms": 15.14}, "gstreamer_push": {"avg_ms": 1.25, "count": 211, "max_ms": 1.474, "p95_ms": 1.324}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.863, "count": 211, "max_ms": 71.157, "p95_ms": 67.548}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.202, "count": 211, "max_ms": 90.123, "p95_ms": 87.579}, "yolo11_640_fire_11class": {"avg_ms": 72.663, "count": 211, "max_ms": 80.579, "p95_ms": 77.614}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.483, "count": 211, "max_ms": 85.752, "p95_ms": 81.685}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.832, "count": 211, "max_ms": 114.44, "p95_ms": 109.817}, "read_interval": {"avg_ms": 120.265, "count": 210, "max_ms": 131.357, "p95_ms": 125.413}, "triton_wait": {"avg_ms": 82.214, "count": 211, "max_ms": 90.134, "p95_ms": 87.591}, "visualize": {"avg_ms": 3.552, "count": 211, "max_ms": 4.511, "p95_ms": 3.805}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.899, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 4980}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9460.309, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55e17efa87c0] error while decoding MB 69 27, bytestream -23", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 82, "h264_decode_error_count": 82, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:51:31+09:00", "last_error_summary": "[h264 @ 0x55e17efa87c0] error while decoding MB 69 27, bytestream -23", "last_exit_at": "2026-07-02T08:35:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 972.582, "last_started_at": "2026-07-02T08:35:19+09:00", "pid": 50641, "process": {"exists": false, "pid": 50641}, "restart_count": 81, "restart_count_per_hour": 7, "rtsp_timeout_count": 146, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 82}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.996, "sample_count": 2556, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 597701, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:51:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.083, "last_frame_pushed_at": "2026-07-02T08:51:31+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-02T08:51:31+09:00", "mjpeg": {"age_sec": 0.063, "exists": true, "mtime": "2026-07-02T08:51:31.547759+09:00", "mtime_epoch": 1782949891.547759, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 237072}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T08:51:31+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1150.68, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 269, "pushed_frames": 269, "read_frames": 269, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.974, "pushed": 8.974, "read": 8.974}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.743, "count": 270, "max_ms": 7.868, "p95_ms": 6.639}, "gstreamer_push": {"avg_ms": 1.613, "count": 269, "max_ms": 6.012, "p95_ms": 3.824}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.193, "count": 270, "max_ms": 76.783, "p95_ms": 73.053}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.475, "count": 270, "max_ms": 98.485, "p95_ms": 91.595}, "yolo11_640_fire_11class": {"avg_ms": 72.585, "count": 270, "max_ms": 91.055, "p95_ms": 84.323}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.932, "count": 270, "max_ms": 91.165, "p95_ms": 86.728}}, "truncated_count": 0}, "processing_total": {"avg_ms": 95.185, "count": 269, "max_ms": 113.324, "p95_ms": 106.192}, "read_interval": {"avg_ms": 111.572, "count": 269, "max_ms": 129.069, "p95_ms": 123.796}, "triton_wait": {"avg_ms": 80.549, "count": 270, "max_ms": 98.497, "p95_ms": 91.606}, "visualize": {"avg_ms": 3.683, "count": 270, "max_ms": 5.824, "p95_ms": 3.891}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.975, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 4980}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9459.824, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T08:51:31+09:00", "last_error_summary": "[h264 @ 0x55e17efa87c0] error while decoding MB 69 27, bytestream -23", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 82, "h264_decode_error_count": 82, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:51:31+09:00", "last_error_summary": "[h264 @ 0x55e17efa87c0] error while decoding MB 69 27, bytestream -23", "last_exit_at": "2026-07-02T08:51:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 972.604, "last_started_at": "2026-07-02T08:35:19+09:00", "pid": 50641, "process": {"exists": false, "pid": 50641}, "restart_count": 81, "restart_count_per_hour": 7, "rtsp_timeout_count": 146, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 82}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.996, "sample_count": 2557, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 597701, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:51:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-02T08:51:31+09:00", "last_frame_read_age_sec": 0.086, "last_frame_read_at": "2026-07-02T08:51:31+09:00", "mjpeg": {"age_sec": 0.085, "exists": true, "mtime": "2026-07-02T08:51:31.547759+09:00", "mtime_epoch": 1782949891.547759, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 237072}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T08:51:31+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1150.68, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 270, "pushed_frames": 270, "read_frames": 269, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 9.001, "pushed": 9.001, "read": 8.968}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.744, "count": 269, "max_ms": 7.868, "p95_ms": 6.639}, "gstreamer_push": {"avg_ms": 1.612, "count": 270, "max_ms": 6.012, "p95_ms": 3.824}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.204, "count": 269, "max_ms": 76.783, "p95_ms": 73.053}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.446, "count": 269, "max_ms": 98.485, "p95_ms": 91.595}, "yolo11_640_fire_11class": {"avg_ms": 72.554, "count": 269, "max_ms": 91.055, "p95_ms": 84.323}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.902, "count": 269, "max_ms": 91.165, "p95_ms": 86.728}}, "truncated_count": 0}, "processing_total": {"avg_ms": 95.115, "count": 270, "max_ms": 113.324, "p95_ms": 106.192}, "read_interval": {"avg_ms": 111.572, "count": 269, "max_ms": 129.069, "p95_ms": 123.796}, "triton_wait": {"avg_ms": 80.519, "count": 269, "max_ms": 98.497, "p95_ms": 91.606}, "visualize": {"avg_ms": 3.684, "count": 269, "max_ms": 5.824, "p95_ms": 3.891}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 4980}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9459.82, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 82, "h264_decode_error_count": 82, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:51:31+09:00", "last_error_summary": "[h264 @ 0x55e17efa87c0] error while decoding MB 69 27, bytestream -23", "last_exit_at": "2026-07-02T08:51:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 972.613, "last_started_at": "2026-07-02T08:35:19+09:00", "pid": 50641, "process": {"exists": false, "pid": 50641}, "restart_count": 81, "restart_count_per_hour": 7, "rtsp_timeout_count": 146, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 82}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.996, "sample_count": 2556, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 597701, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:51:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.017, "last_frame_pushed_at": "2026-07-02T08:51:31+09:00", "last_frame_read_age_sec": 0.096, "last_frame_read_at": "2026-07-02T08:51:31+09:00", "mjpeg": {"age_sec": 0.094, "exists": true, "mtime": "2026-07-02T08:51:31.547759+09:00", "mtime_epoch": 1782949891.547759, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 237072}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T08:51:31+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1150.68, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 269, "pushed_frames": 269, "read_frames": 269, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.971, "pushed": 8.971, "read": 8.971}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.744, "count": 269, "max_ms": 7.868, "p95_ms": 6.639}, "gstreamer_push": {"avg_ms": 1.613, "count": 269, "max_ms": 6.012, "p95_ms": 3.824}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.204, "count": 269, "max_ms": 76.783, "p95_ms": 73.053}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.446, "count": 269, "max_ms": 98.485, "p95_ms": 91.595}, "yolo11_640_fire_11class": {"avg_ms": 72.554, "count": 269, "max_ms": 91.055, "p95_ms": 84.323}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.902, "count": 269, "max_ms": 91.165, "p95_ms": 86.728}}, "truncated_count": 0}, "processing_total": {"avg_ms": 95.087, "count": 269, "max_ms": 113.324, "p95_ms": 106.192}, "read_interval": {"avg_ms": 111.572, "count": 269, "max_ms": 129.069, "p95_ms": 123.796}, "triton_wait": {"avg_ms": 80.519, "count": 269, "max_ms": 98.497, "p95_ms": 91.606}, "visualize": {"avg_ms": 3.684, "count": 269, "max_ms": 5.824, "p95_ms": 3.891}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.987, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 4980}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9639.934, "limit_mb": 8796093022207.996}, "detail": {"pid": 51114, "restart_count": 82, "start_count": 83}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 82, "h264_decode_error_count": 82, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:51:31+09:00", "last_error_summary": "[h264 @ 0x55e17efa87c0] error while decoding MB 69 27, bytestream -23", "last_exit_at": "2026-07-02T08:51:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T08:51:34+09:00", "pid": 51114, "process": {"exists": true, "fd_count": 5, "hwm_mb": 209.469, "name": "ffmpeg", "pid": 51114, "rss_mb": 209.469, "state": "S (sleeping)", "threads": 106}, "restart_count": 82, "restart_count_per_hour": 8, "rtsp_timeout_count": 146, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 83}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.976, "sample_count": 2516, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 597701, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:51:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.968, "last_frame_pushed_at": "2026-07-02T08:51:31+09:00", "last_frame_read_age_sec": 5.046, "last_frame_read_at": "2026-07-02T08:51:31+09:00", "mjpeg": {"age_sec": 4.949, "exists": true, "mtime": "2026-07-02T08:51:31.643761+09:00", "mtime_epoch": 1782949891.643761, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 236943}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T08:51:36+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1150.68, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 228, "pushed_frames": 228, "read_frames": 227, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.619, "pushed": 7.619, "read": 7.585}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.767, "count": 228, "max_ms": 7.868, "p95_ms": 6.658}, "gstreamer_push": {"avg_ms": 1.623, "count": 228, "max_ms": 6.012, "p95_ms": 3.835}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.975, "count": 228, "max_ms": 76.218, "p95_ms": 70.784}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.897, "count": 228, "max_ms": 98.485, "p95_ms": 91.597}, "yolo11_640_fire_11class": {"avg_ms": 71.176, "count": 228, "max_ms": 91.055, "p95_ms": 84.226}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.739, "count": 228, "max_ms": 91.165, "p95_ms": 87.253}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.601, "count": 228, "max_ms": 113.324, "p95_ms": 106.408}, "read_interval": {"avg_ms": 110.064, "count": 227, "max_ms": 129.069, "p95_ms": 123.796}, "triton_wait": {"avg_ms": 78.982, "count": 228, "max_ms": 98.497, "p95_ms": 91.61}, "visualize": {"avg_ms": 3.695, "count": 228, "max_ms": 5.824, "p95_ms": 3.932}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.926, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 4993}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9468.605, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x561c2b47ccc0] error while decoding MB 58 37, bytestream -21 [in#0/rtsp @ 0x561c2b37fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561c2b37fe00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 83, "h264_decode_error_count": 83, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:54:46+09:00", "last_error_summary": "[h264 @ 0x561c2b47ccc0] error while decoding MB 58 37, bytestream -21 [in#0/rtsp @ 0x561c2b37fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561c2b37fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T08:51:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 191.537, "last_started_at": "2026-07-02T08:51:34+09:00", "pid": 51114, "process": {"exists": false, "pid": 51114}, "restart_count": 82, "restart_count_per_hour": 8, "rtsp_timeout_count": 148, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 83}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.965, "sample_count": 2383, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 599160, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:54:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.078, "last_frame_pushed_at": "2026-07-02T08:54:46+09:00", "last_frame_read_age_sec": 0.066, "last_frame_read_at": "2026-07-02T08:54:46+09:00", "mjpeg": {"age_sec": 0.058, "exists": true, "mtime": "2026-07-02T08:54:46.071413+09:00", "mtime_epoch": 1782950086.0714133, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 218176}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T08:54:46+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1148.418, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 60, "pushed_frames": 60, "read_frames": 60, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.003, "pushed": 2.003, "read": 2.003}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.733, "count": 60, "max_ms": 6.676, "p95_ms": 6.554}, "gstreamer_push": {"avg_ms": 1.413, "count": 60, "max_ms": 3.933, "p95_ms": 1.482}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.798, "count": 60, "max_ms": 81.464, "p95_ms": 69.917}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 83.477, "count": 60, "max_ms": 95.301, "p95_ms": 92.997}, "yolo11_640_fire_11class": {"avg_ms": 75.195, "count": 60, "max_ms": 88.509, "p95_ms": 84.664}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.531, "count": 60, "max_ms": 88.497, "p95_ms": 87.927}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.098, "count": 60, "max_ms": 109.66, "p95_ms": 107.209}, "read_interval": {"avg_ms": 499.948, "count": 60, "max_ms": 14469.449, "p95_ms": 126.658}, "triton_wait": {"avg_ms": 83.672, "count": 60, "max_ms": 95.313, "p95_ms": 93.011}, "visualize": {"avg_ms": 3.656, "count": 60, "max_ms": 4.306, "p95_ms": 3.813}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.954, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 4993}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9467.93, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T08:54:46+09:00", "last_error_summary": "[h264 @ 0x561c2b47ccc0] error while decoding MB 58 37, bytestream -21 [in#0/rtsp @ 0x561c2b37fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561c2b37fe00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 83, "h264_decode_error_count": 83, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:54:46+09:00", "last_error_summary": "[h264 @ 0x561c2b47ccc0] error while decoding MB 58 37, bytestream -21 [in#0/rtsp @ 0x561c2b37fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561c2b37fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T08:54:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 191.59, "last_started_at": "2026-07-02T08:51:34+09:00", "pid": 51114, "process": {"exists": false, "pid": 51114}, "restart_count": 82, "restart_count_per_hour": 8, "rtsp_timeout_count": 148, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 83}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.976, "sample_count": 2384, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 599161, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:54:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T08:54:46+09:00", "last_frame_read_age_sec": 0.119, "last_frame_read_at": "2026-07-02T08:54:46+09:00", "mjpeg": {"age_sec": 0.11, "exists": true, "mtime": "2026-07-02T08:54:46.071413+09:00", "mtime_epoch": 1782950086.0714133, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 218176}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T08:54:46+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1148.418, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 60, "pushed_frames": 60, "read_frames": 60, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.001, "pushed": 2.001, "read": 2.001}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.734, "count": 60, "max_ms": 6.676, "p95_ms": 6.554}, "gstreamer_push": {"avg_ms": 1.411, "count": 60, "max_ms": 3.933, "p95_ms": 1.482}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.698, "count": 60, "max_ms": 81.464, "p95_ms": 69.917}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 83.498, "count": 60, "max_ms": 96.555, "p95_ms": 92.997}, "yolo11_640_fire_11class": {"avg_ms": 75.173, "count": 60, "max_ms": 87.207, "p95_ms": 84.664}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.497, "count": 60, "max_ms": 88.355, "p95_ms": 86.926}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.119, "count": 60, "max_ms": 110.927, "p95_ms": 107.209}, "read_interval": {"avg_ms": 499.948, "count": 60, "max_ms": 14469.449, "p95_ms": 126.658}, "triton_wait": {"avg_ms": 83.693, "count": 60, "max_ms": 96.567, "p95_ms": 93.011}, "visualize": {"avg_ms": 3.657, "count": 60, "max_ms": 4.306, "p95_ms": 3.813}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.99, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 4993}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9467.922, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 83, "h264_decode_error_count": 83, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:54:46+09:00", "last_error_summary": "[h264 @ 0x561c2b47ccc0] error while decoding MB 58 37, bytestream -21 [in#0/rtsp @ 0x561c2b37fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561c2b37fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T08:54:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 191.598, "last_started_at": "2026-07-02T08:51:34+09:00", "pid": 51114, "process": {"exists": false, "pid": 51114}, "restart_count": 82, "restart_count_per_hour": 8, "rtsp_timeout_count": 148, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 83}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.976, "sample_count": 2384, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 599161, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:54:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-02T08:54:46+09:00", "last_frame_read_age_sec": 0.127, "last_frame_read_at": "2026-07-02T08:54:46+09:00", "mjpeg": {"age_sec": 0.118, "exists": true, "mtime": "2026-07-02T08:54:46.071413+09:00", "mtime_epoch": 1782950086.0714133, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 218176}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T08:54:46+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1148.418, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 60, "pushed_frames": 60, "read_frames": 60, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.0, "pushed": 2.0, "read": 2.0}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.734, "count": 60, "max_ms": 6.676, "p95_ms": 6.554}, "gstreamer_push": {"avg_ms": 1.411, "count": 60, "max_ms": 3.933, "p95_ms": 1.482}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.698, "count": 60, "max_ms": 81.464, "p95_ms": 69.917}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 83.498, "count": 60, "max_ms": 96.555, "p95_ms": 92.997}, "yolo11_640_fire_11class": {"avg_ms": 75.173, "count": 60, "max_ms": 87.207, "p95_ms": 84.664}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.497, "count": 60, "max_ms": 88.355, "p95_ms": 86.926}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.119, "count": 60, "max_ms": 110.927, "p95_ms": 107.209}, "read_interval": {"avg_ms": 499.948, "count": 60, "max_ms": 14469.449, "p95_ms": 126.658}, "triton_wait": {"avg_ms": 83.693, "count": 60, "max_ms": 96.567, "p95_ms": 93.011}, "visualize": {"avg_ms": 3.657, "count": 60, "max_ms": 4.306, "p95_ms": 3.813}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 4993}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9647.297, "limit_mb": 8796093022207.996}, "detail": {"pid": 51266, "restart_count": 83, "start_count": 84}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 83, "h264_decode_error_count": 83, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:54:46+09:00", "last_error_summary": "[h264 @ 0x561c2b47ccc0] error while decoding MB 58 37, bytestream -21 [in#0/rtsp @ 0x561c2b37fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561c2b37fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T08:54:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T08:54:48+09:00", "pid": 51266, "process": {"exists": true, "fd_count": 5, "hwm_mb": 209.547, "name": "ffmpeg", "pid": 51266, "rss_mb": 209.547, "state": "S (sleeping)", "threads": 106}, "restart_count": 83, "restart_count_per_hour": 9, "rtsp_timeout_count": 148, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 84}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.965, "sample_count": 2346, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 599161, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:54:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.307, "last_frame_pushed_at": "2026-07-02T08:54:46+09:00", "last_frame_read_age_sec": 4.42, "last_frame_read_at": "2026-07-02T08:54:46+09:00", "mjpeg": {"age_sec": 4.288, "exists": true, "mtime": "2026-07-02T08:54:46.195415+09:00", "mtime_epoch": 1782950086.1954155, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 217082}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T08:54:50+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1148.418, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 34, "pushed_frames": 34, "read_frames": 34, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.522, "pushed": 1.522, "read": 1.522}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.78, "count": 34, "max_ms": 6.676, "p95_ms": 6.667}, "gstreamer_push": {"avg_ms": 1.321, "count": 34, "max_ms": 1.495, "p95_ms": 1.482}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.053, "count": 34, "max_ms": 73.174, "p95_ms": 69.917}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.424, "count": 34, "max_ms": 96.555, "p95_ms": 91.024}, "yolo11_640_fire_11class": {"avg_ms": 70.256, "count": 34, "max_ms": 87.207, "p95_ms": 82.526}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.666, "count": 34, "max_ms": 86.421, "p95_ms": 83.7}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.242, "count": 34, "max_ms": 110.927, "p95_ms": 106.353}, "read_interval": {"avg_ms": 789.737, "count": 34, "max_ms": 14469.449, "p95_ms": 8931.659}, "triton_wait": {"avg_ms": 78.759, "count": 34, "max_ms": 96.567, "p95_ms": 91.037}, "visualize": {"avg_ms": 3.674, "count": 34, "max_ms": 4.306, "p95_ms": 3.854}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.34, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3593}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9523.426, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 5, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x563e757f6e00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563e757f6e00] error while decoding MB 0 25, bytestream 438 [h264 @ 0x563e7581f000] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563e7581f000] error while decoding MB 0 39, bytestream 52058 [h264 @ 0x563e7572e6c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563e7572e6c0] error while decoding MB 0 23, bytestream 4562 [h264 @ 0x563e75870300] left block unavailable for reque", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 285, "h264_decode_error_count": 540, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:56:07+09:00", "last_error_summary": "[h264 @ 0x563e757f6e00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563e757f6e00] error while decoding MB 0 25, bytestream 438 [h264 @ 0x563e7581f000] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563e7581f000] error while decoding MB 0 39, bytestream 52058 [h264 @ 0x563e7572e6c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563e7572e6c0] error while decoding MB 0 23, bytestream 4562 [h264 @ 0x563e75870300] left block unavailable for reque", "last_exit_at": "2026-07-02T08:50:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 325.991, "last_started_at": "2026-07-02T08:50:41+09:00", "pid": 51007, "process": {"exists": false, "pid": 51007}, "restart_count": 319, "restart_count_per_hour": 11, "rtsp_timeout_count": 128, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 320}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 124.364, "sample_count": 2373, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 431261, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:56:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.073, "last_frame_pushed_at": "2026-07-02T08:56:07+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-02T08:56:07+09:00", "mjpeg": {"age_sec": 0.055, "exists": true, "mtime": "2026-07-02T08:56:07.264912+09:00", "mtime_epoch": 1782950167.264912, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186486}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:56:07+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5563.449, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 88, "pushed_frames": 88, "read_frames": 88, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.934, "pushed": 2.934, "read": 2.934}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.101, "count": 88, "max_ms": 21.929, "p95_ms": 15.17}, "gstreamer_push": {"avg_ms": 1.251, "count": 88, "max_ms": 1.344, "p95_ms": 1.324}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.776, "count": 88, "max_ms": 74.241, "p95_ms": 67.501}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.943, "count": 88, "max_ms": 92.395, "p95_ms": 85.971}, "yolo11_640_fire_11class": {"avg_ms": 72.287, "count": 88, "max_ms": 80.158, "p95_ms": 78.357}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.08, "count": 88, "max_ms": 81.878, "p95_ms": 79.52}}, "truncated_count": 0}, "processing_total": {"avg_ms": 102.389, "count": 88, "max_ms": 114.79, "p95_ms": 108.547}, "read_interval": {"avg_ms": 341.338, "count": 88, "max_ms": 19801.207, "p95_ms": 125.185}, "triton_wait": {"avg_ms": 79.955, "count": 88, "max_ms": 92.408, "p95_ms": 85.983}, "visualize": {"avg_ms": 3.552, "count": 88, "max_ms": 3.983, "p95_ms": 3.806}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3593}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9522.852, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T08:56:07+09:00", "last_error_summary": "[h264 @ 0x563e757f6e00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563e757f6e00] error while decoding MB 0 25, bytestream 438 [h264 @ 0x563e7581f000] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563e7581f000] error while decoding MB 0 39, bytestream 52058 [h264 @ 0x563e7572e6c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563e7572e6c0] error while decoding MB 0 23, bytestream 4562 [h264 @ 0x563e75870300] left block unavailable for reque", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 285, "h264_decode_error_count": 540, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:56:07+09:00", "last_error_summary": "[h264 @ 0x563e757f6e00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563e757f6e00] error while decoding MB 0 25, bytestream 438 [h264 @ 0x563e7581f000] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563e7581f000] error while decoding MB 0 39, bytestream 52058 [h264 @ 0x563e7572e6c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563e7572e6c0] error while decoding MB 0 23, bytestream 4562 [h264 @ 0x563e75870300] left block unavailable for reque", "last_exit_at": "2026-07-02T08:56:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 326.047, "last_started_at": "2026-07-02T08:50:41+09:00", "pid": 51007, "process": {"exists": false, "pid": 51007}, "restart_count": 319, "restart_count_per_hour": 11, "rtsp_timeout_count": 128, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 320}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 124.364, "sample_count": 2374, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 431262, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:56:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T08:56:07+09:00", "last_frame_read_age_sec": 0.117, "last_frame_read_at": "2026-07-02T08:56:07+09:00", "mjpeg": {"age_sec": 0.111, "exists": true, "mtime": "2026-07-02T08:56:07.264912+09:00", "mtime_epoch": 1782950167.264912, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186486}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:56:07+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5563.449, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 88, "pushed_frames": 88, "read_frames": 87, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.941, "pushed": 2.941, "read": 2.907}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.099, "count": 88, "max_ms": 21.929, "p95_ms": 15.17}, "gstreamer_push": {"avg_ms": 1.251, "count": 88, "max_ms": 1.344, "p95_ms": 1.324}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.8, "count": 88, "max_ms": 74.241, "p95_ms": 67.501}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.981, "count": 88, "max_ms": 92.395, "p95_ms": 86.173}, "yolo11_640_fire_11class": {"avg_ms": 72.309, "count": 88, "max_ms": 80.158, "p95_ms": 78.384}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.13, "count": 88, "max_ms": 81.878, "p95_ms": 79.52}}, "truncated_count": 0}, "processing_total": {"avg_ms": 102.428, "count": 88, "max_ms": 114.79, "p95_ms": 108.696}, "read_interval": {"avg_ms": 343.862, "count": 87, "max_ms": 19801.207, "p95_ms": 125.185}, "triton_wait": {"avg_ms": 79.993, "count": 88, "max_ms": 92.408, "p95_ms": 86.186}, "visualize": {"avg_ms": 3.554, "count": 88, "max_ms": 3.983, "p95_ms": 3.806}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.926, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3593}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9522.902, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 285, "h264_decode_error_count": 540, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:56:07+09:00", "last_error_summary": "[h264 @ 0x563e757f6e00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563e757f6e00] error while decoding MB 0 25, bytestream 438 [h264 @ 0x563e7581f000] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563e7581f000] error while decoding MB 0 39, bytestream 52058 [h264 @ 0x563e7572e6c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563e7572e6c0] error while decoding MB 0 23, bytestream 4562 [h264 @ 0x563e75870300] left block unavailable for reque", "last_exit_at": "2026-07-02T08:56:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 326.055, "last_started_at": "2026-07-02T08:50:41+09:00", "pid": 51007, "process": {"exists": false, "pid": 51007}, "restart_count": 319, "restart_count_per_hour": 11, "rtsp_timeout_count": 128, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 320}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 124.364, "sample_count": 2374, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 431262, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:56:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-02T08:56:07+09:00", "last_frame_read_age_sec": 0.125, "last_frame_read_at": "2026-07-02T08:56:07+09:00", "mjpeg": {"age_sec": 0.119, "exists": true, "mtime": "2026-07-02T08:56:07.264912+09:00", "mtime_epoch": 1782950167.264912, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186486}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:56:07+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5563.449, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 88, "pushed_frames": 88, "read_frames": 87, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.94, "pushed": 2.94, "read": 2.906}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.099, "count": 88, "max_ms": 21.929, "p95_ms": 15.17}, "gstreamer_push": {"avg_ms": 1.251, "count": 88, "max_ms": 1.344, "p95_ms": 1.324}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.8, "count": 88, "max_ms": 74.241, "p95_ms": 67.501}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.981, "count": 88, "max_ms": 92.395, "p95_ms": 86.173}, "yolo11_640_fire_11class": {"avg_ms": 72.309, "count": 88, "max_ms": 80.158, "p95_ms": 78.384}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.13, "count": 88, "max_ms": 81.878, "p95_ms": 79.52}}, "truncated_count": 0}, "processing_total": {"avg_ms": 102.428, "count": 88, "max_ms": 114.79, "p95_ms": 108.696}, "read_interval": {"avg_ms": 343.862, "count": 87, "max_ms": 19801.207, "p95_ms": 125.185}, "triton_wait": {"avg_ms": 79.993, "count": 88, "max_ms": 92.408, "p95_ms": 86.186}, "visualize": {"avg_ms": 3.554, "count": 88, "max_ms": 3.983, "p95_ms": 3.806}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.934, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3593}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9633.277, "limit_mb": 8796093022207.996}, "detail": {"pid": 51418, "restart_count": 320, "start_count": 321}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 285, "h264_decode_error_count": 540, "h264_decode_error_count_per_hour": 38, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:56:07+09:00", "last_error_summary": "[h264 @ 0x563e757f6e00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563e757f6e00] error while decoding MB 0 25, bytestream 438 [h264 @ 0x563e7581f000] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563e7581f000] error while decoding MB 0 39, bytestream 52058 [h264 @ 0x563e7572e6c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563e7572e6c0] error while decoding MB 0 23, bytestream 4562 [h264 @ 0x563e75870300] left block unavailable for reque", "last_exit_at": "2026-07-02T08:56:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T08:56:10+09:00", "pid": 51418, "process": {"exists": true, "fd_count": 5, "hwm_mb": 144.277, "name": "ffmpeg", "pid": 51418, "rss_mb": 144.277, "state": "S (sleeping)", "threads": 106}, "restart_count": 320, "restart_count_per_hour": 12, "rtsp_timeout_count": 128, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 321}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 124.413, "sample_count": 2334, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 431262, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:56:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.719, "last_frame_pushed_at": "2026-07-02T08:56:07+09:00", "last_frame_read_age_sec": 4.83, "last_frame_read_at": "2026-07-02T08:56:07+09:00", "mjpeg": {"age_sec": 4.7, "exists": true, "mtime": "2026-07-02T08:56:07.388914+09:00", "mtime_epoch": 1782950167.388914, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187652}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T08:56:12+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5563.449, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 48, "pushed_frames": 48, "read_frames": 47, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.604, "pushed": 1.604, "read": 1.57}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.118, "count": 48, "max_ms": 21.929, "p95_ms": 15.17}, "gstreamer_push": {"avg_ms": 1.254, "count": 48, "max_ms": 1.344, "p95_ms": 1.327}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.941, "count": 48, "max_ms": 74.241, "p95_ms": 67.501}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.929, "count": 48, "max_ms": 92.395, "p95_ms": 88.909}, "yolo11_640_fire_11class": {"avg_ms": 71.586, "count": 48, "max_ms": 80.158, "p95_ms": 80.061}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.688, "count": 48, "max_ms": 81.878, "p95_ms": 79.52}}, "truncated_count": 0}, "processing_total": {"avg_ms": 102.4, "count": 48, "max_ms": 114.79, "p95_ms": 111.091}, "read_interval": {"avg_ms": 536.095, "count": 47, "max_ms": 19801.207, "p95_ms": 128.194}, "triton_wait": {"avg_ms": 79.941, "count": 48, "max_ms": 92.408, "p95_ms": 88.921}, "visualize": {"avg_ms": 3.578, "count": 48, "max_ms": 3.922, "p95_ms": 3.806}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.928, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 5006}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9477.434, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55c34f3fa880] error while decoding MB 70 59, bytestream -27 [in#0/rtsp @ 0x55c34f2bce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c34f2bce00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 84, "h264_decode_error_count": 84, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:58:26+09:00", "last_error_summary": "[h264 @ 0x55c34f3fa880] error while decoding MB 70 59, bytestream -27 [in#0/rtsp @ 0x55c34f2bce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c34f2bce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T08:54:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 217.997, "last_started_at": "2026-07-02T08:54:48+09:00", "pid": 51266, "process": {"exists": false, "pid": 51266}, "restart_count": 83, "restart_count_per_hour": 8, "rtsp_timeout_count": 150, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 84}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.968, "sample_count": 2136, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 600809, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:58:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.081, "last_frame_pushed_at": "2026-07-02T08:58:26+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-02T08:58:26+09:00", "mjpeg": {"age_sec": 0.06, "exists": true, "mtime": "2026-07-02T08:58:26.419452+09:00", "mtime_epoch": 1782950306.4194524, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 221765}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T08:58:26+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1149.191, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.667, "pushed": 0.667, "read": 0.722}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.796, "count": 12, "max_ms": 6.487, "p95_ms": 6.487}, "gstreamer_push": {"avg_ms": 1.54, "count": 12, "max_ms": 3.551, "p95_ms": 3.551}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.902, "count": 12, "max_ms": 64.482, "p95_ms": 64.482}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.943, "count": 12, "max_ms": 99.735, "p95_ms": 99.735}, "yolo11_640_fire_11class": {"avg_ms": 66.688, "count": 12, "max_ms": 86.93, "p95_ms": 86.93}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.242, "count": 12, "max_ms": 84.217, "p95_ms": 84.217}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.688, "count": 12, "max_ms": 114.253, "p95_ms": 114.253}, "read_interval": {"avg_ms": 2626.212, "count": 13, "max_ms": 16781.41, "p95_ms": 16781.41}, "triton_wait": {"avg_ms": 74.956, "count": 12, "max_ms": 99.748, "p95_ms": 99.748}, "visualize": {"avg_ms": 3.698, "count": 12, "max_ms": 3.881, "p95_ms": 3.881}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.003, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 5006}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9476.992, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T08:58:26+09:00", "last_error_summary": "[h264 @ 0x55c34f3fa880] error while decoding MB 70 59, bytestream -27 [in#0/rtsp @ 0x55c34f2bce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c34f2bce00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 84, "h264_decode_error_count": 84, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:58:26+09:00", "last_error_summary": "[h264 @ 0x55c34f3fa880] error while decoding MB 70 59, bytestream -27 [in#0/rtsp @ 0x55c34f2bce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c34f2bce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T08:58:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 218.04, "last_started_at": "2026-07-02T08:54:48+09:00", "pid": 51266, "process": {"exists": false, "pid": 51266}, "restart_count": 83, "restart_count_per_hour": 8, "rtsp_timeout_count": 150, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 84}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.968, "sample_count": 2137, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 600810, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:58:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T08:58:26+09:00", "last_frame_read_age_sec": 0.108, "last_frame_read_at": "2026-07-02T08:58:26+09:00", "mjpeg": {"age_sec": 0.103, "exists": true, "mtime": "2026-07-02T08:58:26.419452+09:00", "mtime_epoch": 1782950306.4194524, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 221765}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T08:58:26+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1149.195, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.72, "pushed": 0.72, "read": 0.72}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.78, "count": 13, "max_ms": 6.487, "p95_ms": 6.487}, "gstreamer_push": {"avg_ms": 1.531, "count": 13, "max_ms": 3.551, "p95_ms": 3.551}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.114, "count": 13, "max_ms": 64.482, "p95_ms": 64.482}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.77, "count": 13, "max_ms": 99.735, "p95_ms": 99.735}, "yolo11_640_fire_11class": {"avg_ms": 66.969, "count": 13, "max_ms": 86.93, "p95_ms": 86.93}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.017, "count": 13, "max_ms": 84.217, "p95_ms": 84.217}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.521, "count": 13, "max_ms": 114.253, "p95_ms": 114.253}, "read_interval": {"avg_ms": 2626.212, "count": 13, "max_ms": 16781.41, "p95_ms": 16781.41}, "triton_wait": {"avg_ms": 75.784, "count": 13, "max_ms": 99.748, "p95_ms": 99.748}, "visualize": {"avg_ms": 3.709, "count": 13, "max_ms": 3.881, "p95_ms": 3.881}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.045, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 5006}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9476.883, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 84, "h264_decode_error_count": 84, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:58:26+09:00", "last_error_summary": "[h264 @ 0x55c34f3fa880] error while decoding MB 70 59, bytestream -27 [in#0/rtsp @ 0x55c34f2bce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c34f2bce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T08:58:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 218.047, "last_started_at": "2026-07-02T08:54:48+09:00", "pid": 51266, "process": {"exists": false, "pid": 51266}, "restart_count": 83, "restart_count_per_hour": 8, "rtsp_timeout_count": 150, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 84}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.968, "sample_count": 2137, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 600810, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:58:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-02T08:58:26+09:00", "last_frame_read_age_sec": 0.115, "last_frame_read_at": "2026-07-02T08:58:26+09:00", "mjpeg": {"age_sec": 0.11, "exists": true, "mtime": "2026-07-02T08:58:26.419452+09:00", "mtime_epoch": 1782950306.4194524, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 221765}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T08:58:26+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1149.195, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.72, "pushed": 0.72, "read": 0.72}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.78, "count": 13, "max_ms": 6.487, "p95_ms": 6.487}, "gstreamer_push": {"avg_ms": 1.531, "count": 13, "max_ms": 3.551, "p95_ms": 3.551}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.114, "count": 13, "max_ms": 64.482, "p95_ms": 64.482}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.77, "count": 13, "max_ms": 99.735, "p95_ms": 99.735}, "yolo11_640_fire_11class": {"avg_ms": 66.969, "count": 13, "max_ms": 86.93, "p95_ms": 86.93}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.017, "count": 13, "max_ms": 84.217, "p95_ms": 84.217}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.521, "count": 13, "max_ms": 114.253, "p95_ms": 114.253}, "read_interval": {"avg_ms": 2626.212, "count": 13, "max_ms": 16781.41, "p95_ms": 16781.41}, "triton_wait": {"avg_ms": 75.784, "count": 13, "max_ms": 99.748, "p95_ms": 99.748}, "visualize": {"avg_ms": 3.709, "count": 13, "max_ms": 3.881, "p95_ms": 3.881}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.052, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 5006}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9612.398, "limit_mb": 8796093022207.996}, "detail": {"pid": 51525, "restart_count": 84, "start_count": 85}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 84, "h264_decode_error_count": 84, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T08:58:26+09:00", "last_error_summary": "[h264 @ 0x55c34f3fa880] error while decoding MB 70 59, bytestream -27 [in#0/rtsp @ 0x55c34f2bce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c34f2bce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T08:58:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T08:58:29+09:00", "pid": 51525, "process": {"exists": true, "fd_count": 5, "hwm_mb": 161.539, "name": "ffmpeg", "pid": 51525, "rss_mb": 161.539, "state": "R (running)", "threads": 106}, "restart_count": 84, "restart_count_per_hour": 9, "rtsp_timeout_count": 150, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 85}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.982, "sample_count": 2096, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 600810, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T08:58:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.723, "last_frame_pushed_at": "2026-07-02T08:58:26+09:00", "last_frame_read_age_sec": 4.825, "last_frame_read_at": "2026-07-02T08:58:26+09:00", "mjpeg": {"age_sec": 4.704, "exists": true, "mtime": "2026-07-02T08:58:26.535455+09:00", "mtime_epoch": 1782950306.5354545, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 221540}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T08:58:31+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1149.195, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.571, "pushed": 0.571, "read": 0.571}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.78, "count": 13, "max_ms": 6.487, "p95_ms": 6.487}, "gstreamer_push": {"avg_ms": 1.531, "count": 13, "max_ms": 3.551, "p95_ms": 3.551}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.114, "count": 13, "max_ms": 64.482, "p95_ms": 64.482}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.77, "count": 13, "max_ms": 99.735, "p95_ms": 99.735}, "yolo11_640_fire_11class": {"avg_ms": 66.969, "count": 13, "max_ms": 86.93, "p95_ms": 86.93}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.017, "count": 13, "max_ms": 84.217, "p95_ms": 84.217}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.521, "count": 13, "max_ms": 114.253, "p95_ms": 114.253}, "read_interval": {"avg_ms": 2626.212, "count": 13, "max_ms": 16781.41, "p95_ms": 16781.41}, "triton_wait": {"avg_ms": 75.784, "count": 13, "max_ms": 99.748, "p95_ms": 99.748}, "visualize": {"avg_ms": 3.709, "count": 13, "max_ms": 3.881, "p95_ms": 3.881}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.763, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3613}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9526.848, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55b8396f8fc0] error while decoding MB 64 37, bytestream -26 [h264 @ 0x55b839743f40] error while decoding MB 8 43, bytestream -22 [h264 @ 0x55b8396b6640] reference picture missing during reorder [h264 @ 0x55b8396b6640] Missing reference picture, default is 65568 [h264 @ 0x55b8396a45c0] mmco: unref short failure [h264 @ 0x55b8396a45c0] illegal short term buffer state detected [h264 @ 0x55b83968e940] cabac decode of qscale diff failed at 33 34 [h264 @ 0x55b83968e940] error while decoding ", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 286, "h264_decode_error_count": 544, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:01:08+09:00", "last_error_summary": "[h264 @ 0x55b8396f8fc0] error while decoding MB 64 37, bytestream -26 [h264 @ 0x55b839743f40] error while decoding MB 8 43, bytestream -22 [h264 @ 0x55b8396b6640] reference picture missing during reorder [h264 @ 0x55b8396b6640] Missing reference picture, default is 65568 [h264 @ 0x55b8396a45c0] mmco: unref short failure [h264 @ 0x55b8396a45c0] illegal short term buffer state detected [h264 @ 0x55b83968e940] cabac decode of qscale diff failed at 33 34 [h264 @ 0x55b83968e940] error while decoding ", "last_exit_at": "2026-07-02T08:56:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 297.913, "last_started_at": "2026-07-02T08:56:10+09:00", "pid": 51418, "process": {"exists": false, "pid": 51418}, "restart_count": 320, "restart_count_per_hour": 11, "rtsp_timeout_count": 130, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 321}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 109.58, "sample_count": 2412, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 433674, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:01:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.088, "last_frame_pushed_at": "2026-07-02T09:01:07+09:00", "last_frame_read_age_sec": 0.075, "last_frame_read_at": "2026-07-02T09:01:07+09:00", "mjpeg": {"age_sec": 0.071, "exists": true, "mtime": "2026-07-02T09:01:07.930367+09:00", "mtime_epoch": 1782950467.9303665, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185697}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T09:01:08+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5564.48, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 89, "pushed_frames": 89, "read_frames": 90, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.967, "pushed": 2.967, "read": 3.0}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.038, "count": 89, "max_ms": 17.022, "p95_ms": 14.614}, "gstreamer_push": {"avg_ms": 1.253, "count": 89, "max_ms": 1.446, "p95_ms": 1.347}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.308, "count": 89, "max_ms": 67.784, "p95_ms": 67.13}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.662, "count": 89, "max_ms": 91.59, "p95_ms": 89.267}, "yolo11_640_fire_11class": {"avg_ms": 71.309, "count": 89, "max_ms": 80.463, "p95_ms": 78.278}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.053, "count": 89, "max_ms": 83.1, "p95_ms": 82.305}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.224, "count": 89, "max_ms": 113.129, "p95_ms": 111.107}, "read_interval": {"avg_ms": 333.653, "count": 90, "max_ms": 10248.276, "p95_ms": 126.513}, "triton_wait": {"avg_ms": 81.674, "count": 89, "max_ms": 91.602, "p95_ms": 89.279}, "visualize": {"avg_ms": 3.575, "count": 89, "max_ms": 4.354, "p95_ms": 3.941}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3613}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9526.105, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T09:01:08+09:00", "last_error_summary": "[h264 @ 0x55b8396f8fc0] error while decoding MB 64 37, bytestream -26 [h264 @ 0x55b839743f40] error while decoding MB 8 43, bytestream -22 [h264 @ 0x55b8396b6640] reference picture missing during reorder [h264 @ 0x55b8396b6640] Missing reference picture, default is 65568 [h264 @ 0x55b8396a45c0] mmco: unref short failure [h264 @ 0x55b8396a45c0] illegal short term buffer state detected [h264 @ 0x55b83968e940] cabac decode of qscale diff failed at 33 34 [h264 @ 0x55b83968e940] error while decoding ", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 286, "h264_decode_error_count": 544, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:01:08+09:00", "last_error_summary": "[h264 @ 0x55b8396f8fc0] error while decoding MB 64 37, bytestream -26 [h264 @ 0x55b839743f40] error while decoding MB 8 43, bytestream -22 [h264 @ 0x55b8396b6640] reference picture missing during reorder [h264 @ 0x55b8396b6640] Missing reference picture, default is 65568 [h264 @ 0x55b8396a45c0] mmco: unref short failure [h264 @ 0x55b8396a45c0] illegal short term buffer state detected [h264 @ 0x55b83968e940] cabac decode of qscale diff failed at 33 34 [h264 @ 0x55b83968e940] error while decoding ", "last_exit_at": "2026-07-02T09:01:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 297.952, "last_started_at": "2026-07-02T08:56:10+09:00", "pid": 51418, "process": {"exists": false, "pid": 51418}, "restart_count": 320, "restart_count_per_hour": 11, "rtsp_timeout_count": 130, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 321}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 109.58, "sample_count": 2413, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 433675, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:01:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T09:01:08+09:00", "last_frame_read_age_sec": 0.114, "last_frame_read_at": "2026-07-02T09:01:07+09:00", "mjpeg": {"age_sec": 0.11, "exists": true, "mtime": "2026-07-02T09:01:07.930367+09:00", "mtime_epoch": 1782950467.9303665, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185697}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T09:01:08+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5564.48, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 90, "pushed_frames": 90, "read_frames": 89, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.005, "pushed": 3.005, "read": 2.971}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.037, "count": 90, "max_ms": 17.022, "p95_ms": 14.614}, "gstreamer_push": {"avg_ms": 1.254, "count": 90, "max_ms": 1.446, "p95_ms": 1.349}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.328, "count": 90, "max_ms": 67.784, "p95_ms": 67.13}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.689, "count": 90, "max_ms": 91.59, "p95_ms": 89.267}, "yolo11_640_fire_11class": {"avg_ms": 71.28, "count": 90, "max_ms": 80.463, "p95_ms": 78.278}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.111, "count": 90, "max_ms": 83.1, "p95_ms": 82.305}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.253, "count": 90, "max_ms": 113.129, "p95_ms": 111.107}, "read_interval": {"avg_ms": 336.204, "count": 89, "max_ms": 10248.276, "p95_ms": 126.513}, "triton_wait": {"avg_ms": 81.701, "count": 90, "max_ms": 91.602, "p95_ms": 89.279}, "visualize": {"avg_ms": 3.573, "count": 90, "max_ms": 4.354, "p95_ms": 3.941}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.952, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3613}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9525.98, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 286, "h264_decode_error_count": 544, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:01:08+09:00", "last_error_summary": "[h264 @ 0x55b8396f8fc0] error while decoding MB 64 37, bytestream -26 [h264 @ 0x55b839743f40] error while decoding MB 8 43, bytestream -22 [h264 @ 0x55b8396b6640] reference picture missing during reorder [h264 @ 0x55b8396b6640] Missing reference picture, default is 65568 [h264 @ 0x55b8396a45c0] mmco: unref short failure [h264 @ 0x55b8396a45c0] illegal short term buffer state detected [h264 @ 0x55b83968e940] cabac decode of qscale diff failed at 33 34 [h264 @ 0x55b83968e940] error while decoding ", "last_exit_at": "2026-07-02T09:01:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 297.961, "last_started_at": "2026-07-02T08:56:10+09:00", "pid": 51418, "process": {"exists": false, "pid": 51418}, "restart_count": 320, "restart_count_per_hour": 11, "rtsp_timeout_count": 130, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 321}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 109.58, "sample_count": 2413, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 433675, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:01:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-02T09:01:08+09:00", "last_frame_read_age_sec": 0.124, "last_frame_read_at": "2026-07-02T09:01:07+09:00", "mjpeg": {"age_sec": 0.119, "exists": true, "mtime": "2026-07-02T09:01:07.930367+09:00", "mtime_epoch": 1782950467.9303665, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185697}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T09:01:08+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5564.48, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 90, "pushed_frames": 90, "read_frames": 89, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.004, "pushed": 3.004, "read": 2.971}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.037, "count": 90, "max_ms": 17.022, "p95_ms": 14.614}, "gstreamer_push": {"avg_ms": 1.254, "count": 90, "max_ms": 1.446, "p95_ms": 1.349}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.328, "count": 90, "max_ms": 67.784, "p95_ms": 67.13}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.689, "count": 90, "max_ms": 91.59, "p95_ms": 89.267}, "yolo11_640_fire_11class": {"avg_ms": 71.28, "count": 90, "max_ms": 80.463, "p95_ms": 78.278}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.111, "count": 90, "max_ms": 83.1, "p95_ms": 82.305}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.253, "count": 90, "max_ms": 113.129, "p95_ms": 111.107}, "read_interval": {"avg_ms": 336.204, "count": 89, "max_ms": 10248.276, "p95_ms": 126.513}, "triton_wait": {"avg_ms": 81.701, "count": 90, "max_ms": 91.602, "p95_ms": 89.279}, "visualize": {"avg_ms": 3.573, "count": 90, "max_ms": 4.354, "p95_ms": 3.941}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.961, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3613}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9639.16, "limit_mb": 8796093022207.996}, "detail": {"pid": 51677, "restart_count": 321, "start_count": 322}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 286, "h264_decode_error_count": 544, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:01:08+09:00", "last_error_summary": "[h264 @ 0x55b8396f8fc0] error while decoding MB 64 37, bytestream -26 [h264 @ 0x55b839743f40] error while decoding MB 8 43, bytestream -22 [h264 @ 0x55b8396b6640] reference picture missing during reorder [h264 @ 0x55b8396b6640] Missing reference picture, default is 65568 [h264 @ 0x55b8396a45c0] mmco: unref short failure [h264 @ 0x55b8396a45c0] illegal short term buffer state detected [h264 @ 0x55b83968e940] cabac decode of qscale diff failed at 33 34 [h264 @ 0x55b83968e940] error while decoding ", "last_exit_at": "2026-07-02T09:01:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T09:01:10+09:00", "pid": 51677, "process": {"exists": true, "fd_count": 5, "hwm_mb": 147.387, "name": "ffmpeg", "pid": 51677, "rss_mb": 147.387, "state": "S (sleeping)", "threads": 106}, "restart_count": 321, "restart_count_per_hour": 12, "rtsp_timeout_count": 130, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 322}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 109.58, "sample_count": 2411, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 433675, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:01:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.338, "last_frame_pushed_at": "2026-07-02T09:01:08+09:00", "last_frame_read_age_sec": 4.447, "last_frame_read_at": "2026-07-02T09:01:07+09:00", "mjpeg": {"age_sec": 4.323, "exists": true, "mtime": "2026-07-02T09:01:08.050369+09:00", "mtime_epoch": 1782950468.0503688, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186444}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T09:01:12+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5564.48, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 59, "pushed_frames": 59, "read_frames": 59, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.738, "pushed": 2.738, "read": 2.738}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.031, "count": 59, "max_ms": 15.496, "p95_ms": 14.547}, "gstreamer_push": {"avg_ms": 1.271, "count": 59, "max_ms": 1.443, "p95_ms": 1.355}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.417, "count": 59, "max_ms": 67.784, "p95_ms": 67.13}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.438, "count": 59, "max_ms": 89.499, "p95_ms": 89.33}, "yolo11_640_fire_11class": {"avg_ms": 71.602, "count": 59, "max_ms": 80.463, "p95_ms": 80.099}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.233, "count": 59, "max_ms": 83.1, "p95_ms": 83.023}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.098, "count": 59, "max_ms": 112.318, "p95_ms": 112.032}, "read_interval": {"avg_ms": 447.025, "count": 59, "max_ms": 10248.276, "p95_ms": 127.555}, "triton_wait": {"avg_ms": 82.45, "count": 59, "max_ms": 89.511, "p95_ms": 89.341}, "visualize": {"avg_ms": 3.608, "count": 59, "max_ms": 4.354, "p95_ms": 3.966}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.551, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 5019}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9468.234, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x56393a85f300] error while decoding MB 49 36, bytestream -7 [in#0/rtsp @ 0x563939aa1e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563939aa1e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 85, "h264_decode_error_count": 85, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:01:51+09:00", "last_error_summary": "[h264 @ 0x56393a85f300] error while decoding MB 49 36, bytestream -7 [in#0/rtsp @ 0x563939aa1e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563939aa1e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T08:58:29+09:00", "last_returncode": 0, "last_stable_duration_sec": 202.502, "last_started_at": "2026-07-02T08:58:29+09:00", "pid": 51525, "process": {"exists": false, "pid": 51525}, "restart_count": 84, "restart_count_per_hour": 9, "rtsp_timeout_count": 152, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 85}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.875, "sample_count": 2092, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 602378, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:01:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-02T09:01:51+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-02T09:01:51+09:00", "mjpeg": {"age_sec": 0.058, "exists": true, "mtime": "2026-07-02T09:01:51.683151+09:00", "mtime_epoch": 1782950511.683151, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 238796}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T09:01:51+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1149.969, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 26, "pushed_frames": 26, "read_frames": 26, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.867, "pushed": 0.867, "read": 0.867}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.688, "count": 26, "max_ms": 6.236, "p95_ms": 5.799}, "gstreamer_push": {"avg_ms": 1.791, "count": 26, "max_ms": 3.955, "p95_ms": 3.873}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.187, "count": 26, "max_ms": 60.109, "p95_ms": 50.098}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 86.468, "count": 26, "max_ms": 92.949, "p95_ms": 92.395}, "yolo11_640_fire_11class": {"avg_ms": 72.765, "count": 26, "max_ms": 83.138, "p95_ms": 82.038}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.382, "count": 26, "max_ms": 83.941, "p95_ms": 83.242}}, "truncated_count": 0}, "processing_total": {"avg_ms": 101.005, "count": 26, "max_ms": 108.596, "p95_ms": 106.589}, "read_interval": {"avg_ms": 1154.859, "count": 26, "max_ms": 16337.049, "p95_ms": 10867.085}, "triton_wait": {"avg_ms": 86.481, "count": 26, "max_ms": 92.961, "p95_ms": 92.408}, "visualize": {"avg_ms": 3.682, "count": 26, "max_ms": 4.21, "p95_ms": 3.874}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.985, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 5019}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9467.809, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T09:01:51+09:00", "last_error_summary": "[h264 @ 0x56393a85f300] error while decoding MB 49 36, bytestream -7 [in#0/rtsp @ 0x563939aa1e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563939aa1e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 85, "h264_decode_error_count": 85, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:01:51+09:00", "last_error_summary": "[h264 @ 0x56393a85f300] error while decoding MB 49 36, bytestream -7 [in#0/rtsp @ 0x563939aa1e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563939aa1e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T09:01:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 202.55, "last_started_at": "2026-07-02T08:58:29+09:00", "pid": 51525, "process": {"exists": false, "pid": 51525}, "restart_count": 84, "restart_count_per_hour": 9, "rtsp_timeout_count": 152, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 85}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.875, "sample_count": 2093, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 602379, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:01:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T09:01:51+09:00", "last_frame_read_age_sec": 0.113, "last_frame_read_at": "2026-07-02T09:01:51+09:00", "mjpeg": {"age_sec": 0.105, "exists": true, "mtime": "2026-07-02T09:01:51.683151+09:00", "mtime_epoch": 1782950511.683151, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 238796}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T09:01:51+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1149.969, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 26, "pushed_frames": 26, "read_frames": 25, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.869, "pushed": 0.869, "read": 0.836}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.687, "count": 26, "max_ms": 6.236, "p95_ms": 5.799}, "gstreamer_push": {"avg_ms": 1.791, "count": 26, "max_ms": 3.955, "p95_ms": 3.873}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.367, "count": 26, "max_ms": 60.109, "p95_ms": 50.098}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 86.386, "count": 26, "max_ms": 92.949, "p95_ms": 91.708}, "yolo11_640_fire_11class": {"avg_ms": 72.724, "count": 26, "max_ms": 83.138, "p95_ms": 82.038}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.397, "count": 26, "max_ms": 83.941, "p95_ms": 83.501}}, "truncated_count": 0}, "processing_total": {"avg_ms": 100.934, "count": 26, "max_ms": 108.596, "p95_ms": 106.113}, "read_interval": {"avg_ms": 1195.996, "count": 25, "max_ms": 16337.049, "p95_ms": 10867.085}, "triton_wait": {"avg_ms": 86.399, "count": 26, "max_ms": 92.961, "p95_ms": 91.722}, "visualize": {"avg_ms": 3.686, "count": 26, "max_ms": 4.21, "p95_ms": 3.874}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.908, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 5019}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9467.793, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 85, "h264_decode_error_count": 85, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:01:51+09:00", "last_error_summary": "[h264 @ 0x56393a85f300] error while decoding MB 49 36, bytestream -7 [in#0/rtsp @ 0x563939aa1e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563939aa1e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T09:01:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 202.557, "last_started_at": "2026-07-02T08:58:29+09:00", "pid": 51525, "process": {"exists": false, "pid": 51525}, "restart_count": 84, "restart_count_per_hour": 9, "rtsp_timeout_count": 152, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": false, "start_count": 85}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.875, "sample_count": 2093, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 602379, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:01:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-02T09:01:51+09:00", "last_frame_read_age_sec": 0.12, "last_frame_read_at": "2026-07-02T09:01:51+09:00", "mjpeg": {"age_sec": 0.112, "exists": true, "mtime": "2026-07-02T09:01:51.683151+09:00", "mtime_epoch": 1782950511.683151, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 238796}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T09:01:51+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1149.969, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 26, "pushed_frames": 26, "read_frames": 25, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.869, "pushed": 0.869, "read": 0.836}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.687, "count": 26, "max_ms": 6.236, "p95_ms": 5.799}, "gstreamer_push": {"avg_ms": 1.791, "count": 26, "max_ms": 3.955, "p95_ms": 3.873}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 43.367, "count": 26, "max_ms": 60.109, "p95_ms": 50.098}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 86.386, "count": 26, "max_ms": 92.949, "p95_ms": 91.708}, "yolo11_640_fire_11class": {"avg_ms": 72.724, "count": 26, "max_ms": 83.138, "p95_ms": 82.038}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 74.397, "count": 26, "max_ms": 83.941, "p95_ms": 83.501}}, "truncated_count": 0}, "processing_total": {"avg_ms": 100.934, "count": 26, "max_ms": 108.596, "p95_ms": 106.113}, "read_interval": {"avg_ms": 1195.996, "count": 25, "max_ms": 16337.049, "p95_ms": 10867.085}, "triton_wait": {"avg_ms": 86.399, "count": 26, "max_ms": 92.961, "p95_ms": 91.722}, "visualize": {"avg_ms": 3.686, "count": 26, "max_ms": 4.21, "p95_ms": 3.874}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.915, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 5019}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9592.465, "limit_mb": 8796093022207.996}, "detail": {"pid": 51784, "restart_count": 85, "start_count": 86}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 85, "h264_decode_error_count": 85, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:01:51+09:00", "last_error_summary": "[h264 @ 0x56393a85f300] error while decoding MB 49 36, bytestream -7 [in#0/rtsp @ 0x563939aa1e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563939aa1e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T09:01:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T09:01:54+09:00", "pid": 51784, "process": {"exists": true, "fd_count": 5, "hwm_mb": 156.637, "name": "ffmpeg", "pid": 51784, "rss_mb": 156.473, "state": "R (running)", "threads": 49}, "restart_count": 85, "restart_count_per_hour": 10, "rtsp_timeout_count": 152, "rtsp_timeout_count_per_hour": 16, "rtsp_tolerant_mode": false, "running": true, "start_count": 86}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.877, "sample_count": 2056, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 602379, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:01:51+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.448, "last_frame_pushed_at": "2026-07-02T09:01:51+09:00", "last_frame_read_age_sec": 4.555, "last_frame_read_at": "2026-07-02T09:01:51+09:00", "mjpeg": {"age_sec": 4.431, "exists": true, "mtime": "2026-07-02T09:01:51.799153+09:00", "mtime_epoch": 1782950511.799153, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 236701}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T09:01:56+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1149.969, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.629, "pushed": 0.629, "read": 0.629}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.713, "count": 14, "max_ms": 6.236, "p95_ms": 6.236}, "gstreamer_push": {"avg_ms": 1.648, "count": 14, "max_ms": 3.955, "p95_ms": 3.955}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 44.576, "count": 14, "max_ms": 60.109, "p95_ms": 60.109}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 83.76, "count": 14, "max_ms": 90.262, "p95_ms": 90.262}, "yolo11_640_fire_11class": {"avg_ms": 69.046, "count": 14, "max_ms": 80.942, "p95_ms": 80.942}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.707, "count": 14, "max_ms": 83.501, "p95_ms": 83.501}}, "truncated_count": 0}, "processing_total": {"avg_ms": 98.306, "count": 14, "max_ms": 105.578, "p95_ms": 105.578}, "read_interval": {"avg_ms": 2041.004, "count": 14, "max_ms": 16337.049, "p95_ms": 16337.049}, "triton_wait": {"avg_ms": 83.773, "count": 14, "max_ms": 90.273, "p95_ms": 90.273}, "visualize": {"avg_ms": 3.698, "count": 14, "max_ms": 4.21, "p95_ms": 4.21}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.262, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 5032}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9480.605, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55668f646fc0] error while decoding MB 79 43, bytestream -43 [in#0/rtsp @ 0x55668f4a2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55668f4a2e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 86, "h264_decode_error_count": 86, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:05:18+09:00", "last_error_summary": "[h264 @ 0x55668f646fc0] error while decoding MB 79 43, bytestream -43 [in#0/rtsp @ 0x55668f4a2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55668f4a2e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T09:01:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 204.089, "last_started_at": "2026-07-02T09:01:54+09:00", "pid": 51784, "process": {"exists": false, "pid": 51784}, "restart_count": 85, "restart_count_per_hour": 10, "rtsp_timeout_count": 154, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 86}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.99, "sample_count": 2156, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 603887, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:05:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.078, "last_frame_pushed_at": "2026-07-02T09:05:18+09:00", "last_frame_read_age_sec": 0.063, "last_frame_read_at": "2026-07-02T09:05:18+09:00", "mjpeg": {"age_sec": 0.059, "exists": true, "mtime": "2026-07-02T09:05:18.258832+09:00", "mtime_epoch": 1782950718.2588317, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 249506}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T09:05:18+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1151.199, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 22, "pushed_frames": 22, "read_frames": 22, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.736, "pushed": 0.736, "read": 0.736}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.621, "count": 22, "max_ms": 5.865, "p95_ms": 5.755}, "gstreamer_push": {"avg_ms": 1.33, "count": 22, "max_ms": 1.45, "p95_ms": 1.443}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.864, "count": 22, "max_ms": 63.146, "p95_ms": 60.404}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.512, "count": 22, "max_ms": 92.544, "p95_ms": 92.218}, "yolo11_640_fire_11class": {"avg_ms": 68.704, "count": 22, "max_ms": 83.19, "p95_ms": 82.693}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.507, "count": 22, "max_ms": 86.784, "p95_ms": 84.676}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.245, "count": 22, "max_ms": 107.107, "p95_ms": 106.389}, "read_interval": {"avg_ms": 1360.777, "count": 22, "max_ms": 16589.547, "p95_ms": 11251.074}, "triton_wait": {"avg_ms": 76.73, "count": 22, "max_ms": 92.556, "p95_ms": 92.229}, "visualize": {"avg_ms": 3.729, "count": 22, "max_ms": 4.178, "p95_ms": 3.84}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.897, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 5032}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9479.801, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T09:05:18+09:00", "last_error_summary": "[h264 @ 0x55668f646fc0] error while decoding MB 79 43, bytestream -43 [in#0/rtsp @ 0x55668f4a2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55668f4a2e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 86, "h264_decode_error_count": 86, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:05:18+09:00", "last_error_summary": "[h264 @ 0x55668f646fc0] error while decoding MB 79 43, bytestream -43 [in#0/rtsp @ 0x55668f4a2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55668f4a2e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T09:05:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 204.141, "last_started_at": "2026-07-02T09:01:54+09:00", "pid": 51784, "process": {"exists": false, "pid": 51784}, "restart_count": 85, "restart_count_per_hour": 10, "rtsp_timeout_count": 154, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 86}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.998, "sample_count": 2157, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 603888, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:05:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T09:05:18+09:00", "last_frame_read_age_sec": 0.115, "last_frame_read_at": "2026-07-02T09:05:18+09:00", "mjpeg": {"age_sec": 0.111, "exists": true, "mtime": "2026-07-02T09:05:18.258832+09:00", "mtime_epoch": 1782950718.2588317, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 249506}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T09:05:18+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1151.199, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 23, "pushed_frames": 23, "read_frames": 22, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.768, "pushed": 0.768, "read": 0.735}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.623, "count": 23, "max_ms": 5.865, "p95_ms": 5.755}, "gstreamer_push": {"avg_ms": 1.337, "count": 23, "max_ms": 1.479, "p95_ms": 1.45}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.702, "count": 23, "max_ms": 66.128, "p95_ms": 63.146}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.206, "count": 23, "max_ms": 92.544, "p95_ms": 92.471}, "yolo11_640_fire_11class": {"avg_ms": 69.447, "count": 23, "max_ms": 85.78, "p95_ms": 83.19}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.171, "count": 23, "max_ms": 86.784, "p95_ms": 85.768}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.936, "count": 23, "max_ms": 107.126, "p95_ms": 107.107}, "read_interval": {"avg_ms": 1360.777, "count": 22, "max_ms": 16589.547, "p95_ms": 11251.074}, "triton_wait": {"avg_ms": 77.415, "count": 23, "max_ms": 92.556, "p95_ms": 92.484}, "visualize": {"avg_ms": 3.729, "count": 23, "max_ms": 4.178, "p95_ms": 3.84}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.949, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 5032}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9479.742, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 86, "h264_decode_error_count": 86, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:05:18+09:00", "last_error_summary": "[h264 @ 0x55668f646fc0] error while decoding MB 79 43, bytestream -43 [in#0/rtsp @ 0x55668f4a2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55668f4a2e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T09:05:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 204.148, "last_started_at": "2026-07-02T09:01:54+09:00", "pid": 51784, "process": {"exists": false, "pid": 51784}, "restart_count": 85, "restart_count_per_hour": 10, "rtsp_timeout_count": 154, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": false, "start_count": 86}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.998, "sample_count": 2157, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 603888, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:05:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-02T09:05:18+09:00", "last_frame_read_age_sec": 0.122, "last_frame_read_at": "2026-07-02T09:05:18+09:00", "mjpeg": {"age_sec": 0.118, "exists": true, "mtime": "2026-07-02T09:05:18.258832+09:00", "mtime_epoch": 1782950718.2588317, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 249506}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T09:05:18+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1151.199, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 23, "pushed_frames": 23, "read_frames": 22, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.768, "pushed": 0.768, "read": 0.734}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.623, "count": 23, "max_ms": 5.865, "p95_ms": 5.755}, "gstreamer_push": {"avg_ms": 1.337, "count": 23, "max_ms": 1.479, "p95_ms": 1.45}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.702, "count": 23, "max_ms": 66.128, "p95_ms": 63.146}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.206, "count": 23, "max_ms": 92.544, "p95_ms": 92.471}, "yolo11_640_fire_11class": {"avg_ms": 69.447, "count": 23, "max_ms": 85.78, "p95_ms": 83.19}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.171, "count": 23, "max_ms": 86.784, "p95_ms": 85.768}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.936, "count": 23, "max_ms": 107.126, "p95_ms": 107.107}, "read_interval": {"avg_ms": 1360.777, "count": 22, "max_ms": 16589.547, "p95_ms": 11251.074}, "triton_wait": {"avg_ms": 77.415, "count": 23, "max_ms": 92.556, "p95_ms": 92.484}, "visualize": {"avg_ms": 3.729, "count": 23, "max_ms": 4.178, "p95_ms": 3.84}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.956, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 5032}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9500.613, "limit_mb": 8796093022207.996}, "detail": {"pid": 51936, "restart_count": 86, "start_count": 87}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 86, "h264_decode_error_count": 86, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:05:18+09:00", "last_error_summary": "[h264 @ 0x55668f646fc0] error while decoding MB 79 43, bytestream -43 [in#0/rtsp @ 0x55668f4a2e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55668f4a2e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T09:05:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T09:05:20+09:00", "pid": 51936, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.875, "name": "ffmpeg", "pid": 51936, "rss_mb": 57.875, "state": "S (sleeping)", "threads": 1}, "restart_count": 86, "restart_count_per_hour": 11, "rtsp_timeout_count": 154, "rtsp_timeout_count_per_hour": 18, "rtsp_tolerant_mode": false, "running": true, "start_count": 87}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.861, "sample_count": 2122, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 603888, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:05:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.286, "last_frame_pushed_at": "2026-07-02T09:05:18+09:00", "last_frame_read_age_sec": 4.395, "last_frame_read_at": "2026-07-02T09:05:18+09:00", "mjpeg": {"age_sec": 4.268, "exists": true, "mtime": "2026-07-02T09:05:18.382834+09:00", "mtime_epoch": 1782950718.382834, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 247421}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T09:05:22+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1151.199, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.633, "pushed": 0.633, "read": 0.633}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.66, "count": 14, "max_ms": 5.865, "p95_ms": 5.865}, "gstreamer_push": {"avg_ms": 1.358, "count": 14, "max_ms": 1.479, "p95_ms": 1.479}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.051, "count": 14, "max_ms": 66.128, "p95_ms": 66.128}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.981, "count": 14, "max_ms": 92.471, "p95_ms": 92.471}, "yolo11_640_fire_11class": {"avg_ms": 63.017, "count": 14, "max_ms": 85.78, "p95_ms": 85.78}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.496, "count": 14, "max_ms": 85.768, "p95_ms": 85.768}}, "truncated_count": 0}, "processing_total": {"avg_ms": 83.911, "count": 14, "max_ms": 107.126, "p95_ms": 107.126}, "read_interval": {"avg_ms": 2069.714, "count": 14, "max_ms": 16589.547, "p95_ms": 16589.547}, "triton_wait": {"avg_ms": 69.316, "count": 14, "max_ms": 92.484, "p95_ms": 92.484}, "visualize": {"avg_ms": 3.705, "count": 14, "max_ms": 3.84, "p95_ms": 3.84}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.121, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3639}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9529.074, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5627054b9240] cabac decode of qscale diff failed at 18 17 [h264 @ 0x5627054b9240] error while decoding MB 18 17, bytestream 1572 [h264 @ 0x5627054e1500] error while decoding MB 38 22, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 287, "h264_decode_error_count": 547, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:07:14+09:00", "last_error_summary": "[h264 @ 0x5627054b9240] cabac decode of qscale diff failed at 18 17 [h264 @ 0x5627054b9240] error while decoding MB 18 17, bytestream 1572 [h264 @ 0x5627054e1500] error while decoding MB 38 22, bytestream -9", "last_exit_at": "2026-07-02T09:01:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 364.347, "last_started_at": "2026-07-02T09:01:10+09:00", "pid": 51677, "process": {"exists": false, "pid": 51677}, "restart_count": 321, "restart_count_per_hour": 11, "rtsp_timeout_count": 130, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 322}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 110.145, "sample_count": 2529, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 436785, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:07:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.064, "last_frame_pushed_at": "2026-07-02T09:07:14+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-02T09:07:14+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-02T09:07:14.672893+09:00", "mtime_epoch": 1782950834.6728933, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189228}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T09:07:14+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5566.027, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 226, "pushed_frames": 227, "read_frames": 227, "select_timeouts": 3, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.533, "pushed": 7.567, "read": 7.567}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.083, "count": 226, "max_ms": 16.833, "p95_ms": 14.97}, "gstreamer_push": {"avg_ms": 1.247, "count": 227, "max_ms": 1.659, "p95_ms": 1.305}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.932, "count": 226, "max_ms": 75.257, "p95_ms": 67.726}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.135, "count": 226, "max_ms": 96.878, "p95_ms": 85.911}, "yolo11_640_fire_11class": {"avg_ms": 71.494, "count": 226, "max_ms": 85.711, "p95_ms": 76.227}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.741, "count": 226, "max_ms": 86.298, "p95_ms": 78.968}}, "truncated_count": 0}, "processing_total": {"avg_ms": 102.724, "count": 226, "max_ms": 119.27, "p95_ms": 108.559}, "read_interval": {"avg_ms": 132.404, "count": 227, "max_ms": 3290.332, "p95_ms": 124.303}, "triton_wait": {"avg_ms": 80.15, "count": 226, "max_ms": 96.89, "p95_ms": 85.923}, "visualize": {"avg_ms": 3.55, "count": 226, "max_ms": 4.262, "p95_ms": 3.78}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3639}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9528.531, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T09:07:14+09:00", "last_error_summary": "[h264 @ 0x5627054b9240] cabac decode of qscale diff failed at 18 17 [h264 @ 0x5627054b9240] error while decoding MB 18 17, bytestream 1572 [h264 @ 0x5627054e1500] error while decoding MB 38 22, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 287, "h264_decode_error_count": 547, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:07:14+09:00", "last_error_summary": "[h264 @ 0x5627054b9240] cabac decode of qscale diff failed at 18 17 [h264 @ 0x5627054b9240] error while decoding MB 18 17, bytestream 1572 [h264 @ 0x5627054e1500] error while decoding MB 38 22, bytestream -9", "last_exit_at": "2026-07-02T09:07:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 364.413, "last_started_at": "2026-07-02T09:01:10+09:00", "pid": 51677, "process": {"exists": false, "pid": 51677}, "restart_count": 321, "restart_count_per_hour": 11, "rtsp_timeout_count": 130, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 322}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 110.145, "sample_count": 2529, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 436786, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:07:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T09:07:14+09:00", "last_frame_read_age_sec": 0.118, "last_frame_read_at": "2026-07-02T09:07:14+09:00", "mjpeg": {"age_sec": 0.113, "exists": true, "mtime": "2026-07-02T09:07:14.672893+09:00", "mtime_epoch": 1782950834.6728933, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189228}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T09:07:14+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5566.027, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 227, "pushed_frames": 227, "read_frames": 226, "select_timeouts": 3, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.581, "pushed": 7.581, "read": 7.547}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.083, "count": 227, "max_ms": 16.833, "p95_ms": 14.97}, "gstreamer_push": {"avg_ms": 1.247, "count": 227, "max_ms": 1.659, "p95_ms": 1.305}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.984, "count": 227, "max_ms": 75.257, "p95_ms": 67.755}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.163, "count": 227, "max_ms": 96.878, "p95_ms": 86.025}, "yolo11_640_fire_11class": {"avg_ms": 71.518, "count": 227, "max_ms": 85.711, "p95_ms": 76.269}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.797, "count": 227, "max_ms": 86.298, "p95_ms": 79.176}}, "truncated_count": 0}, "processing_total": {"avg_ms": 102.751, "count": 227, "max_ms": 119.27, "p95_ms": 108.598}, "read_interval": {"avg_ms": 132.453, "count": 226, "max_ms": 3290.332, "p95_ms": 124.303}, "triton_wait": {"avg_ms": 80.178, "count": 227, "max_ms": 96.89, "p95_ms": 86.037}, "visualize": {"avg_ms": 3.55, "count": 227, "max_ms": 4.262, "p95_ms": 3.78}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.944, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3639}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9528.539, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 287, "h264_decode_error_count": 547, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:07:14+09:00", "last_error_summary": "[h264 @ 0x5627054b9240] cabac decode of qscale diff failed at 18 17 [h264 @ 0x5627054b9240] error while decoding MB 18 17, bytestream 1572 [h264 @ 0x5627054e1500] error while decoding MB 38 22, bytestream -9", "last_exit_at": "2026-07-02T09:07:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 364.422, "last_started_at": "2026-07-02T09:01:10+09:00", "pid": 51677, "process": {"exists": false, "pid": 51677}, "restart_count": 321, "restart_count_per_hour": 11, "rtsp_timeout_count": 130, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 322}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 110.145, "sample_count": 2529, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 436786, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:07:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-02T09:07:14+09:00", "last_frame_read_age_sec": 0.126, "last_frame_read_at": "2026-07-02T09:07:14+09:00", "mjpeg": {"age_sec": 0.121, "exists": true, "mtime": "2026-07-02T09:07:14.672893+09:00", "mtime_epoch": 1782950834.6728933, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189228}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T09:07:14+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5566.027, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 227, "pushed_frames": 227, "read_frames": 226, "select_timeouts": 3, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.579, "pushed": 7.579, "read": 7.545}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.083, "count": 227, "max_ms": 16.833, "p95_ms": 14.97}, "gstreamer_push": {"avg_ms": 1.247, "count": 227, "max_ms": 1.659, "p95_ms": 1.305}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.984, "count": 227, "max_ms": 75.257, "p95_ms": 67.755}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.163, "count": 227, "max_ms": 96.878, "p95_ms": 86.025}, "yolo11_640_fire_11class": {"avg_ms": 71.518, "count": 227, "max_ms": 85.711, "p95_ms": 76.269}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.797, "count": 227, "max_ms": 86.298, "p95_ms": 79.176}}, "truncated_count": 0}, "processing_total": {"avg_ms": 102.751, "count": 227, "max_ms": 119.27, "p95_ms": 108.598}, "read_interval": {"avg_ms": 132.453, "count": 226, "max_ms": 3290.332, "p95_ms": 124.303}, "triton_wait": {"avg_ms": 80.178, "count": 227, "max_ms": 96.89, "p95_ms": 86.037}, "visualize": {"avg_ms": 3.55, "count": 227, "max_ms": 4.262, "p95_ms": 3.78}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.953, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3639}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9634.301, "limit_mb": 8796093022207.996}, "detail": {"pid": 52088, "restart_count": 322, "start_count": 323}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 287, "h264_decode_error_count": 547, "h264_decode_error_count_per_hour": 37, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:07:14+09:00", "last_error_summary": "[h264 @ 0x5627054b9240] cabac decode of qscale diff failed at 18 17 [h264 @ 0x5627054b9240] error while decoding MB 18 17, bytestream 1572 [h264 @ 0x5627054e1500] error while decoding MB 38 22, bytestream -9", "last_exit_at": "2026-07-02T09:07:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T09:07:17+09:00", "pid": 52088, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.688, "name": "ffmpeg", "pid": 52088, "rss_mb": 136.688, "state": "S (sleeping)", "threads": 106}, "restart_count": 322, "restart_count_per_hour": 12, "rtsp_timeout_count": 130, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 323}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 110.016, "sample_count": 2489, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 436786, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:07:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.784, "last_frame_pushed_at": "2026-07-02T09:07:14+09:00", "last_frame_read_age_sec": 4.895, "last_frame_read_at": "2026-07-02T09:07:14+09:00", "mjpeg": {"age_sec": 4.767, "exists": true, "mtime": "2026-07-02T09:07:14.796896+09:00", "mtime_epoch": 1782950834.7968955, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189382}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T09:07:19+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5566.027, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 187, "pushed_frames": 187, "read_frames": 186, "select_timeouts": 3, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.245, "pushed": 6.245, "read": 6.211}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.078, "count": 187, "max_ms": 16.833, "p95_ms": 14.97}, "gstreamer_push": {"avg_ms": 1.246, "count": 187, "max_ms": 1.659, "p95_ms": 1.305}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.811, "count": 187, "max_ms": 75.257, "p95_ms": 67.47}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.914, "count": 187, "max_ms": 96.878, "p95_ms": 85.911}, "yolo11_640_fire_11class": {"avg_ms": 71.257, "count": 187, "max_ms": 85.711, "p95_ms": 76.651}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.61, "count": 187, "max_ms": 85.564, "p95_ms": 78.905}}, "truncated_count": 0}, "processing_total": {"avg_ms": 102.5, "count": 187, "max_ms": 119.27, "p95_ms": 108.559}, "read_interval": {"avg_ms": 135.247, "count": 186, "max_ms": 3290.332, "p95_ms": 124.618}, "triton_wait": {"avg_ms": 79.929, "count": 187, "max_ms": 96.89, "p95_ms": 85.923}, "visualize": {"avg_ms": 3.557, "count": 187, "max_ms": 4.262, "p95_ms": 3.868}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.946, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 5044}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9465.441, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55f1a8217980] error while decoding MB 45 41, bytestream -5 [in#0/rtsp @ 0x55f1a81f6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f1a81f6e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 87, "h264_decode_error_count": 87, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:08:38+09:00", "last_error_summary": "[h264 @ 0x55f1a8217980] error while decoding MB 45 41, bytestream -5 [in#0/rtsp @ 0x55f1a81f6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f1a81f6e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T09:05:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 197.433, "last_started_at": "2026-07-02T09:05:20+09:00", "pid": 51936, "process": {"exists": false, "pid": 51936}, "restart_count": 86, "restart_count_per_hour": 11, "rtsp_timeout_count": 156, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 87}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.464, "sample_count": 2105, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 605376, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:08:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.08, "last_frame_pushed_at": "2026-07-02T09:08:38+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-02T09:08:38+09:00", "mjpeg": {"age_sec": 0.059, "exists": true, "mtime": "2026-07-02T09:08:38.023446+09:00", "mtime_epoch": 1782950918.0234456, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 240129}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T09:08:38+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1148.879, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 39, "pushed_frames": 39, "read_frames": 39, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.303, "pushed": 1.303, "read": 1.303}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.68, "count": 39, "max_ms": 6.374, "p95_ms": 5.832}, "gstreamer_push": {"avg_ms": 1.647, "count": 39, "max_ms": 3.957, "p95_ms": 3.884}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.429, "count": 39, "max_ms": 64.445, "p95_ms": 60.779}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 87.317, "count": 39, "max_ms": 90.613, "p95_ms": 90.569}, "yolo11_640_fire_11class": {"avg_ms": 78.64, "count": 39, "max_ms": 84.673, "p95_ms": 83.566}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.23, "count": 39, "max_ms": 85.922, "p95_ms": 84.312}}, "truncated_count": 0}, "processing_total": {"avg_ms": 101.801, "count": 39, "max_ms": 106.256, "p95_ms": 105.016}, "read_interval": {"avg_ms": 768.565, "count": 39, "max_ms": 16501.097, "p95_ms": 9071.452}, "triton_wait": {"avg_ms": 87.331, "count": 39, "max_ms": 90.626, "p95_ms": 90.582}, "visualize": {"avg_ms": 3.655, "count": 39, "max_ms": 4.174, "p95_ms": 3.754}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.936, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 5044}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9465.066, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T09:08:38+09:00", "last_error_summary": "[h264 @ 0x55f1a8217980] error while decoding MB 45 41, bytestream -5 [in#0/rtsp @ 0x55f1a81f6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f1a81f6e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 87, "h264_decode_error_count": 87, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:08:38+09:00", "last_error_summary": "[h264 @ 0x55f1a8217980] error while decoding MB 45 41, bytestream -5 [in#0/rtsp @ 0x55f1a81f6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f1a81f6e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T09:08:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 197.479, "last_started_at": "2026-07-02T09:05:20+09:00", "pid": 51936, "process": {"exists": false, "pid": 51936}, "restart_count": 86, "restart_count_per_hour": 11, "rtsp_timeout_count": 156, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 87}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.464, "sample_count": 2105, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 605377, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:08:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T09:08:38+09:00", "last_frame_read_age_sec": 0.112, "last_frame_read_at": "2026-07-02T09:08:38+09:00", "mjpeg": {"age_sec": 0.105, "exists": true, "mtime": "2026-07-02T09:08:38.023446+09:00", "mtime_epoch": 1782950918.0234456, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 240129}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T09:08:38+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1148.879, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 40, "pushed_frames": 40, "read_frames": 39, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.334, "pushed": 1.334, "read": 1.301}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.678, "count": 40, "max_ms": 6.374, "p95_ms": 5.796}, "gstreamer_push": {"avg_ms": 1.639, "count": 40, "max_ms": 3.957, "p95_ms": 3.859}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.477, "count": 40, "max_ms": 64.445, "p95_ms": 57.135}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 87.344, "count": 40, "max_ms": 90.613, "p95_ms": 90.49}, "yolo11_640_fire_11class": {"avg_ms": 78.697, "count": 40, "max_ms": 84.673, "p95_ms": 83.196}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.937, "count": 40, "max_ms": 85.922, "p95_ms": 83.578}}, "truncated_count": 0}, "processing_total": {"avg_ms": 101.818, "count": 40, "max_ms": 106.256, "p95_ms": 104.846}, "read_interval": {"avg_ms": 768.565, "count": 39, "max_ms": 16501.097, "p95_ms": 9071.452}, "triton_wait": {"avg_ms": 87.357, "count": 40, "max_ms": 90.626, "p95_ms": 90.503}, "visualize": {"avg_ms": 3.654, "count": 40, "max_ms": 4.174, "p95_ms": 3.714}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.981, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 5044}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9465.055, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 87, "h264_decode_error_count": 87, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:08:38+09:00", "last_error_summary": "[h264 @ 0x55f1a8217980] error while decoding MB 45 41, bytestream -5 [in#0/rtsp @ 0x55f1a81f6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f1a81f6e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T09:08:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 197.489, "last_started_at": "2026-07-02T09:05:20+09:00", "pid": 51936, "process": {"exists": false, "pid": 51936}, "restart_count": 86, "restart_count_per_hour": 11, "rtsp_timeout_count": 156, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 87}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.464, "sample_count": 2105, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 605377, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:08:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.017, "last_frame_pushed_at": "2026-07-02T09:08:38+09:00", "last_frame_read_age_sec": 0.122, "last_frame_read_at": "2026-07-02T09:08:38+09:00", "mjpeg": {"age_sec": 0.115, "exists": true, "mtime": "2026-07-02T09:08:38.023446+09:00", "mtime_epoch": 1782950918.0234456, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 240129}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T09:08:38+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1148.879, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 40, "pushed_frames": 40, "read_frames": 39, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.334, "pushed": 1.334, "read": 1.3}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.678, "count": 40, "max_ms": 6.374, "p95_ms": 5.796}, "gstreamer_push": {"avg_ms": 1.639, "count": 40, "max_ms": 3.957, "p95_ms": 3.859}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.477, "count": 40, "max_ms": 64.445, "p95_ms": 57.135}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 87.344, "count": 40, "max_ms": 90.613, "p95_ms": 90.49}, "yolo11_640_fire_11class": {"avg_ms": 78.697, "count": 40, "max_ms": 84.673, "p95_ms": 83.196}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.937, "count": 40, "max_ms": 85.922, "p95_ms": 83.578}}, "truncated_count": 0}, "processing_total": {"avg_ms": 101.818, "count": 40, "max_ms": 106.256, "p95_ms": 104.846}, "read_interval": {"avg_ms": 768.565, "count": 39, "max_ms": 16501.097, "p95_ms": 9071.452}, "triton_wait": {"avg_ms": 87.357, "count": 40, "max_ms": 90.626, "p95_ms": 90.503}, "visualize": {"avg_ms": 3.654, "count": 40, "max_ms": 4.174, "p95_ms": 3.714}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 5044}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9486.227, "limit_mb": 8796093022207.996}, "detail": {"pid": 52195, "restart_count": 87, "start_count": 88}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 87, "h264_decode_error_count": 87, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:08:38+09:00", "last_error_summary": "[h264 @ 0x55f1a8217980] error while decoding MB 45 41, bytestream -5 [in#0/rtsp @ 0x55f1a81f6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f1a81f6e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T09:08:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T09:08:40+09:00", "pid": 52195, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.098, "name": "ffmpeg", "pid": 52195, "rss_mb": 58.098, "state": "S (sleeping)", "threads": 1}, "restart_count": 87, "restart_count_per_hour": 12, "rtsp_timeout_count": 156, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": true, "start_count": 88}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.464, "sample_count": 2066, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 605377, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:08:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.587, "last_frame_pushed_at": "2026-07-02T09:08:38+09:00", "last_frame_read_age_sec": 4.692, "last_frame_read_at": "2026-07-02T09:08:38+09:00", "mjpeg": {"age_sec": 4.565, "exists": true, "mtime": "2026-07-02T09:08:38.143450+09:00", "mtime_epoch": 1782950918.1434498, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 237104}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T09:08:42+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1148.879, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.619, "pushed": 0.619, "read": 0.619}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.644, "count": 14, "max_ms": 5.737, "p95_ms": 5.737}, "gstreamer_push": {"avg_ms": 1.666, "count": 14, "max_ms": 3.884, "p95_ms": 3.884}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 46.618, "count": 14, "max_ms": 64.445, "p95_ms": 64.445}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 86.0, "count": 14, "max_ms": 90.49, "p95_ms": 90.49}, "yolo11_640_fire_11class": {"avg_ms": 78.123, "count": 14, "max_ms": 83.566, "p95_ms": 83.566}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.422, "count": 14, "max_ms": 84.312, "p95_ms": 84.312}}, "truncated_count": 0}, "processing_total": {"avg_ms": 100.463, "count": 14, "max_ms": 106.256, "p95_ms": 106.256}, "read_interval": {"avg_ms": 1929.27, "count": 14, "max_ms": 16501.097, "p95_ms": 16501.097}, "triton_wait": {"avg_ms": 86.013, "count": 14, "max_ms": 90.503, "p95_ms": 90.503}, "visualize": {"avg_ms": 3.63, "count": 14, "max_ms": 3.714, "p95_ms": 3.714}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.631, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3663}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9534.086, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x558d62038500] error while decoding MB 56 10, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 288, "h264_decode_error_count": 548, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:12:57+09:00", "last_error_summary": "[h264 @ 0x558d62038500] error while decoding MB 56 10, bytestream -5", "last_exit_at": "2026-07-02T09:07:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 340.025, "last_started_at": "2026-07-02T09:07:17+09:00", "pid": 52088, "process": {"exists": false, "pid": 52088}, "restart_count": 322, "restart_count_per_hour": 11, "rtsp_timeout_count": 130, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 323}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 109.071, "sample_count": 2575, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 439662, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:12:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.068, "last_frame_pushed_at": "2026-07-02T09:12:57+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-02T09:12:57+09:00", "mjpeg": {"age_sec": 0.048, "exists": true, "mtime": "2026-07-02T09:12:57.539738+09:00", "mtime_epoch": 1782951177.539738, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177572}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T09:12:57+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5567.316, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 253, "pushed_frames": 253, "read_frames": 253, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.44, "pushed": 8.44, "read": 8.44}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.284, "count": 253, "max_ms": 16.624, "p95_ms": 15.452}, "gstreamer_push": {"avg_ms": 1.281, "count": 253, "max_ms": 1.531, "p95_ms": 1.386}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.173, "count": 253, "max_ms": 73.637, "p95_ms": 67.011}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.075, "count": 253, "max_ms": 93.124, "p95_ms": 86.122}, "yolo11_640_fire_11class": {"avg_ms": 70.381, "count": 253, "max_ms": 81.078, "p95_ms": 77.361}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.962, "count": 253, "max_ms": 81.783, "p95_ms": 79.533}}, "truncated_count": 0}, "processing_total": {"avg_ms": 102.975, "count": 253, "max_ms": 115.666, "p95_ms": 108.626}, "read_interval": {"avg_ms": 118.688, "count": 253, "max_ms": 130.693, "p95_ms": 124.603}, "triton_wait": {"avg_ms": 80.088, "count": 253, "max_ms": 93.137, "p95_ms": 86.135}, "visualize": {"avg_ms": 3.607, "count": 253, "max_ms": 4.3, "p95_ms": 3.976}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.975, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3663}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9533.539, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T09:12:57+09:00", "last_error_summary": "[h264 @ 0x558d62038500] error while decoding MB 56 10, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 288, "h264_decode_error_count": 548, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:12:57+09:00", "last_error_summary": "[h264 @ 0x558d62038500] error while decoding MB 56 10, bytestream -5", "last_exit_at": "2026-07-02T09:12:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 340.09, "last_started_at": "2026-07-02T09:07:17+09:00", "pid": 52088, "process": {"exists": false, "pid": 52088}, "restart_count": 322, "restart_count_per_hour": 11, "rtsp_timeout_count": 130, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 323}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 109.124, "sample_count": 2575, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 439663, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:12:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T09:12:57+09:00", "last_frame_read_age_sec": 0.12, "last_frame_read_at": "2026-07-02T09:12:57+09:00", "mjpeg": {"age_sec": 0.113, "exists": true, "mtime": "2026-07-02T09:12:57.539738+09:00", "mtime_epoch": 1782951177.539738, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177572}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T09:12:57+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5567.316, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 253, "pushed_frames": 253, "read_frames": 252, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.455, "pushed": 8.455, "read": 8.421}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.283, "count": 253, "max_ms": 16.624, "p95_ms": 15.452}, "gstreamer_push": {"avg_ms": 1.281, "count": 253, "max_ms": 1.531, "p95_ms": 1.386}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.181, "count": 253, "max_ms": 73.637, "p95_ms": 67.02}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.088, "count": 253, "max_ms": 93.124, "p95_ms": 86.35}, "yolo11_640_fire_11class": {"avg_ms": 70.392, "count": 253, "max_ms": 81.078, "p95_ms": 77.361}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.971, "count": 253, "max_ms": 81.817, "p95_ms": 79.533}}, "truncated_count": 0}, "processing_total": {"avg_ms": 102.987, "count": 253, "max_ms": 115.666, "p95_ms": 109.239}, "read_interval": {"avg_ms": 118.672, "count": 252, "max_ms": 130.693, "p95_ms": 124.603}, "triton_wait": {"avg_ms": 80.1, "count": 253, "max_ms": 93.137, "p95_ms": 86.363}, "visualize": {"avg_ms": 3.608, "count": 253, "max_ms": 4.3, "p95_ms": 3.976}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.924, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3663}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9533.453, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 288, "h264_decode_error_count": 548, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:12:57+09:00", "last_error_summary": "[h264 @ 0x558d62038500] error while decoding MB 56 10, bytestream -5", "last_exit_at": "2026-07-02T09:12:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 340.1, "last_started_at": "2026-07-02T09:07:17+09:00", "pid": 52088, "process": {"exists": false, "pid": 52088}, "restart_count": 322, "restart_count_per_hour": 11, "rtsp_timeout_count": 130, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 323}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 109.124, "sample_count": 2575, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 439663, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:12:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.017, "last_frame_pushed_at": "2026-07-02T09:12:57+09:00", "last_frame_read_age_sec": 0.13, "last_frame_read_at": "2026-07-02T09:12:57+09:00", "mjpeg": {"age_sec": 0.123, "exists": true, "mtime": "2026-07-02T09:12:57.539738+09:00", "mtime_epoch": 1782951177.539738, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 177572}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T09:12:57+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5567.316, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 253, "pushed_frames": 253, "read_frames": 252, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.452, "pushed": 8.452, "read": 8.419}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.283, "count": 253, "max_ms": 16.624, "p95_ms": 15.452}, "gstreamer_push": {"avg_ms": 1.281, "count": 253, "max_ms": 1.531, "p95_ms": 1.386}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.181, "count": 253, "max_ms": 73.637, "p95_ms": 67.02}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.088, "count": 253, "max_ms": 93.124, "p95_ms": 86.35}, "yolo11_640_fire_11class": {"avg_ms": 70.392, "count": 253, "max_ms": 81.078, "p95_ms": 77.361}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.971, "count": 253, "max_ms": 81.817, "p95_ms": 79.533}}, "truncated_count": 0}, "processing_total": {"avg_ms": 102.987, "count": 253, "max_ms": 115.666, "p95_ms": 109.239}, "read_interval": {"avg_ms": 118.672, "count": 252, "max_ms": 130.693, "p95_ms": 124.603}, "triton_wait": {"avg_ms": 80.1, "count": 253, "max_ms": 93.137, "p95_ms": 86.363}, "visualize": {"avg_ms": 3.608, "count": 253, "max_ms": 4.3, "p95_ms": 3.976}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.934, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3663}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9639.035, "limit_mb": 8796093022207.996}, "detail": {"pid": 52347, "restart_count": 323, "start_count": 324}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 288, "h264_decode_error_count": 548, "h264_decode_error_count_per_hour": 35, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:12:57+09:00", "last_error_summary": "[h264 @ 0x558d62038500] error while decoding MB 56 10, bytestream -5", "last_exit_at": "2026-07-02T09:13:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T09:13:00+09:00", "pid": 52347, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.582, "name": "ffmpeg", "pid": 52347, "rss_mb": 138.582, "state": "S (sleeping)", "threads": 106}, "restart_count": 323, "restart_count_per_hour": 12, "rtsp_timeout_count": 130, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 324}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 109.071, "sample_count": 2537, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 439663, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:12:57+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.57, "last_frame_pushed_at": "2026-07-02T09:12:57+09:00", "last_frame_read_age_sec": 4.683, "last_frame_read_at": "2026-07-02T09:12:57+09:00", "mjpeg": {"age_sec": 4.552, "exists": true, "mtime": "2026-07-02T09:12:57.663742+09:00", "mtime_epoch": 1782951177.6637416, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 169299}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T09:13:02+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5567.316, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 215, "pushed_frames": 215, "read_frames": 214, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.168, "pushed": 7.168, "read": 7.134}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.294, "count": 214, "max_ms": 16.624, "p95_ms": 15.521}, "gstreamer_push": {"avg_ms": 1.281, "count": 215, "max_ms": 1.531, "p95_ms": 1.396}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.081, "count": 214, "max_ms": 73.385, "p95_ms": 67.439}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.171, "count": 214, "max_ms": 93.124, "p95_ms": 86.513}, "yolo11_640_fire_11class": {"avg_ms": 70.144, "count": 214, "max_ms": 81.078, "p95_ms": 77.361}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.986, "count": 214, "max_ms": 81.817, "p95_ms": 79.584}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.085, "count": 215, "max_ms": 115.666, "p95_ms": 109.42}, "read_interval": {"avg_ms": 118.776, "count": 214, "max_ms": 130.693, "p95_ms": 124.968}, "triton_wait": {"avg_ms": 80.184, "count": 214, "max_ms": 93.137, "p95_ms": 86.527}, "visualize": {"avg_ms": 3.604, "count": 214, "max_ms": 4.3, "p95_ms": 3.972}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3678}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9533.891, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 9, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x558f484d9cc0] error while decoding MB 43 35, bytestream -9 [h264 @ 0x558f4774cd80] left block unavailable for requested intra mode [h264 @ 0x558f4774cd80] error while decoding MB 0 32, bytestream 419 [h264 @ 0x558f47836740] left block unavailable for requested intra mode [h264 @ 0x558f47836740] error while decoding MB 0 34, bytestream 4421 [h264 @ 0x558f4826d140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x558f4826d140] error while decoding MB 0 35, bytestream 5419 ", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 289, "h264_decode_error_count": 557, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:16:52+09:00", "last_error_summary": "[h264 @ 0x558f484d9cc0] error while decoding MB 43 35, bytestream -9 [h264 @ 0x558f4774cd80] left block unavailable for requested intra mode [h264 @ 0x558f4774cd80] error while decoding MB 0 32, bytestream 419 [h264 @ 0x558f47836740] left block unavailable for requested intra mode [h264 @ 0x558f47836740] error while decoding MB 0 34, bytestream 4421 [h264 @ 0x558f4826d140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x558f4826d140] error while decoding MB 0 35, bytestream 5419 ", "last_exit_at": "2026-07-02T09:13:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 232.378, "last_started_at": "2026-07-02T09:13:00+09:00", "pid": 52347, "process": {"exists": false, "pid": 52347}, "restart_count": 323, "restart_count_per_hour": 12, "rtsp_timeout_count": 132, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 324}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 109.414, "sample_count": 2315, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 441431, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:16:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.071, "last_frame_pushed_at": "2026-07-02T09:16:52+09:00", "last_frame_read_age_sec": 0.056, "last_frame_read_at": "2026-07-02T09:16:52+09:00", "mjpeg": {"age_sec": 0.051, "exists": true, "mtime": "2026-07-02T09:16:52.542132+09:00", "mtime_epoch": 1782951412.5421324, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195223}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T09:16:52+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5568.348, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 90, "pushed_frames": 90, "read_frames": 90, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.006, "pushed": 3.006, "read": 3.006}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.04, "count": 90, "max_ms": 15.312, "p95_ms": 14.321}, "gstreamer_push": {"avg_ms": 1.253, "count": 90, "max_ms": 1.372, "p95_ms": 1.31}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.617, "count": 90, "max_ms": 69.483, "p95_ms": 68.476}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.494, "count": 90, "max_ms": 93.263, "p95_ms": 87.428}, "yolo11_640_fire_11class": {"avg_ms": 73.159, "count": 90, "max_ms": 78.568, "p95_ms": 77.507}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.636, "count": 90, "max_ms": 83.017, "p95_ms": 80.728}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.904, "count": 90, "max_ms": 115.399, "p95_ms": 109.816}, "read_interval": {"avg_ms": 332.965, "count": 90, "max_ms": 19272.812, "p95_ms": 126.601}, "triton_wait": {"avg_ms": 82.506, "count": 90, "max_ms": 93.276, "p95_ms": 87.44}, "visualize": {"avg_ms": 3.533, "count": 90, "max_ms": 3.946, "p95_ms": 3.78}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.94, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3678}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9533.371, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T09:16:52+09:00", "last_error_summary": "[h264 @ 0x558f484d9cc0] error while decoding MB 43 35, bytestream -9 [h264 @ 0x558f4774cd80] left block unavailable for requested intra mode [h264 @ 0x558f4774cd80] error while decoding MB 0 32, bytestream 419 [h264 @ 0x558f47836740] left block unavailable for requested intra mode [h264 @ 0x558f47836740] error while decoding MB 0 34, bytestream 4421 [h264 @ 0x558f4826d140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x558f4826d140] error while decoding MB 0 35, bytestream 5419 ", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 289, "h264_decode_error_count": 557, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:16:52+09:00", "last_error_summary": "[h264 @ 0x558f484d9cc0] error while decoding MB 43 35, bytestream -9 [h264 @ 0x558f4774cd80] left block unavailable for requested intra mode [h264 @ 0x558f4774cd80] error while decoding MB 0 32, bytestream 419 [h264 @ 0x558f47836740] left block unavailable for requested intra mode [h264 @ 0x558f47836740] error while decoding MB 0 34, bytestream 4421 [h264 @ 0x558f4826d140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x558f4826d140] error while decoding MB 0 35, bytestream 5419 ", "last_exit_at": "2026-07-02T09:16:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 232.433, "last_started_at": "2026-07-02T09:13:00+09:00", "pid": 52347, "process": {"exists": false, "pid": 52347}, "restart_count": 323, "restart_count_per_hour": 12, "rtsp_timeout_count": 132, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 324}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 109.414, "sample_count": 2316, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 441432, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:16:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T09:16:52+09:00", "last_frame_read_age_sec": 0.111, "last_frame_read_at": "2026-07-02T09:16:52+09:00", "mjpeg": {"age_sec": 0.105, "exists": true, "mtime": "2026-07-02T09:16:52.542132+09:00", "mtime_epoch": 1782951412.5421324, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195223}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T09:16:52+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5568.348, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 91, "pushed_frames": 91, "read_frames": 90, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.034, "pushed": 3.034, "read": 3.001}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.043, "count": 91, "max_ms": 15.312, "p95_ms": 14.321}, "gstreamer_push": {"avg_ms": 1.252, "count": 91, "max_ms": 1.372, "p95_ms": 1.31}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.609, "count": 91, "max_ms": 69.483, "p95_ms": 68.476}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.473, "count": 91, "max_ms": 93.263, "p95_ms": 87.428}, "yolo11_640_fire_11class": {"avg_ms": 73.139, "count": 91, "max_ms": 78.568, "p95_ms": 77.507}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.58, "count": 91, "max_ms": 83.017, "p95_ms": 80.728}}, "truncated_count": 0}, "processing_total": {"avg_ms": 104.884, "count": 91, "max_ms": 115.399, "p95_ms": 109.816}, "read_interval": {"avg_ms": 332.965, "count": 90, "max_ms": 19272.812, "p95_ms": 126.601}, "triton_wait": {"avg_ms": 82.485, "count": 91, "max_ms": 93.276, "p95_ms": 87.44}, "visualize": {"avg_ms": 3.533, "count": 91, "max_ms": 3.946, "p95_ms": 3.78}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3678}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9533.395, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 289, "h264_decode_error_count": 557, "h264_decode_error_count_per_hour": 44, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:16:52+09:00", "last_error_summary": "[h264 @ 0x558f484d9cc0] error while decoding MB 43 35, bytestream -9 [h264 @ 0x558f4774cd80] left block unavailable for requested intra mode [h264 @ 0x558f4774cd80] error while decoding MB 0 32, bytestream 419 [h264 @ 0x558f47836740] left block unavailable for requested intra mode [h264 @ 0x558f47836740] error while decoding MB 0 34, bytestream 4421 [h264 @ 0x558f4826d140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x558f4826d140] error while decoding MB 0 35, bytestream 5419 ", "last_exit_at": "2026-07-02T09:16:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 232.44, "last_started_at": "2026-07-02T09:13:00+09:00", "pid": 52347, "process": {"exists": false, "pid": 52347}, "restart_count": 323, "restart_count_per_hour": 12, "rtsp_timeout_count": 132, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 324}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 109.414, "sample_count": 2316, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 441432, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:16:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-02T09:16:52+09:00", "last_frame_read_age_sec": 0.118, "last_frame_read_at": "2026-07-02T09:16:52+09:00", "mjpeg": {"age_sec": 0.112, "exists": true, "mtime": "2026-07-02T09:16:52.542132+09:00", "mtime_epoch": 1782951412.5421324, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195223}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T09:16:52+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5568.348, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 90, "pushed_frames": 91, "read_frames": 90, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.0, "pushed": 3.033, "read": 3.0}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.029, "count": 90, "max_ms": 15.116, "p95_ms": 14.304}, "gstreamer_push": {"avg_ms": 1.252, "count": 91, "max_ms": 1.372, "p95_ms": 1.31}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 62.858, "count": 90, "max_ms": 69.483, "p95_ms": 68.476}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.72, "count": 90, "max_ms": 93.263, "p95_ms": 87.428}, "yolo11_640_fire_11class": {"avg_ms": 73.391, "count": 90, "max_ms": 78.568, "p95_ms": 77.507}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.838, "count": 90, "max_ms": 83.017, "p95_ms": 80.728}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.118, "count": 90, "max_ms": 115.399, "p95_ms": 109.816}, "read_interval": {"avg_ms": 332.965, "count": 90, "max_ms": 19272.812, "p95_ms": 126.601}, "triton_wait": {"avg_ms": 82.732, "count": 90, "max_ms": 93.276, "p95_ms": 87.44}, "visualize": {"avg_ms": 3.534, "count": 90, "max_ms": 3.946, "p95_ms": 3.78}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260701_144824_21718dd8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 256, "tracking_summary": 3678}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 9641.113, "limit_mb": 8796093022207.996}, "detail": {"pid": 52454, "restart_count": 324, "start_count": 325}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 289, "h264_decode_error_count": 557, "h264_decode_error_count_per_hour": 43, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:16:52+09:00", "last_error_summary": "[h264 @ 0x558f484d9cc0] error while decoding MB 43 35, bytestream -9 [h264 @ 0x558f4774cd80] left block unavailable for requested intra mode [h264 @ 0x558f4774cd80] error while decoding MB 0 32, bytestream 419 [h264 @ 0x558f47836740] left block unavailable for requested intra mode [h264 @ 0x558f47836740] error while decoding MB 0 34, bytestream 4421 [h264 @ 0x558f4826d140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x558f4826d140] error while decoding MB 0 35, bytestream 5419 ", "last_exit_at": "2026-07-02T09:16:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T09:16:54+09:00", "pid": 52454, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.914, "name": "ffmpeg", "pid": 52454, "rss_mb": 138.914, "state": "S (sleeping)", "threads": 106}, "restart_count": 324, "restart_count_per_hour": 13, "rtsp_timeout_count": 132, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 325}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 109.325, "sample_count": 2281, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 441432, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:16:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "21718dd8", "last_frame_pushed_age_sec": 4.109, "last_frame_pushed_at": "2026-07-02T09:16:52+09:00", "last_frame_read_age_sec": 4.214, "last_frame_read_at": "2026-07-02T09:16:52+09:00", "mjpeg": {"age_sec": 4.093, "exists": true, "mtime": "2026-07-02T09:16:52.658135+09:00", "mtime_epoch": 1782951412.6581354, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195285}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T09:16:56+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 5694.262, "name": "celery", "pid": 237, "rss_mb": 5568.348, "state": "S (sleeping)", "threads": 124}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 57, "pushed_frames": 57, "read_frames": 56, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.9, "pushed": 1.9, "read": 1.867}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 14.008, "count": 57, "max_ms": 15.116, "p95_ms": 14.304}, "gstreamer_push": {"avg_ms": 1.249, "count": 57, "max_ms": 1.329, "p95_ms": 1.308}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 63.227, "count": 57, "max_ms": 69.483, "p95_ms": 68.476}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.681, "count": 57, "max_ms": 93.263, "p95_ms": 90.181}, "yolo11_640_fire_11class": {"avg_ms": 72.986, "count": 57, "max_ms": 77.507, "p95_ms": 76.54}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 75.865, "count": 57, "max_ms": 83.017, "p95_ms": 81.106}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.011, "count": 57, "max_ms": 115.399, "p95_ms": 112.172}, "read_interval": {"avg_ms": 462.23, "count": 56, "max_ms": 19272.812, "p95_ms": 127.759}, "triton_wait": {"avg_ms": 82.693, "count": 57, "max_ms": 93.276, "p95_ms": 90.193}, "visualize": {"avg_ms": 3.506, "count": 57, "max_ms": 3.77, "p95_ms": 3.747}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-01T14:48:24+09:00", "state": "running", "task_id": "d8ef5ed0-f7d4-474c-b6eb-bd3d1aee1f7c", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 5082}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9471.883, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x562420808000] error while decoding MB 48 29, bytestream -19 [in#0/rtsp @ 0x5624207c0e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5624207c0e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 88, "h264_decode_error_count": 88, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:17:50+09:00", "last_error_summary": "[h264 @ 0x562420808000] error while decoding MB 48 29, bytestream -19 [in#0/rtsp @ 0x5624207c0e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5624207c0e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T09:08:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 549.875, "last_started_at": "2026-07-02T09:08:40+09:00", "pid": 52195, "process": {"exists": false, "pid": 52195}, "restart_count": 87, "restart_count_per_hour": 10, "rtsp_timeout_count": 158, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 88}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.62, "sample_count": 2413, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 609923, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:17:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-02T09:17:50+09:00", "last_frame_read_age_sec": 0.062, "last_frame_read_at": "2026-07-02T09:17:50+09:00", "mjpeg": {"age_sec": 0.058, "exists": true, "mtime": "2026-07-02T09:17:50.523598+09:00", "mtime_epoch": 1782951470.5235977, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 224567}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T09:17:50+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1153.719, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 23, "pushed_frames": 23, "read_frames": 23, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.767, "pushed": 0.767, "read": 0.767}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.666, "count": 23, "max_ms": 5.867, "p95_ms": 5.73}, "gstreamer_push": {"avg_ms": 1.652, "count": 23, "max_ms": 3.836, "p95_ms": 3.779}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.114, "count": 23, "max_ms": 68.22, "p95_ms": 61.243}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.009, "count": 23, "max_ms": 91.332, "p95_ms": 90.748}, "yolo11_640_fire_11class": {"avg_ms": 70.891, "count": 23, "max_ms": 83.294, "p95_ms": 82.674}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.429, "count": 23, "max_ms": 84.136, "p95_ms": 84.029}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.596, "count": 23, "max_ms": 105.741, "p95_ms": 105.534}, "read_interval": {"avg_ms": 1305.531, "count": 23, "max_ms": 17525.795, "p95_ms": 10164.075}, "triton_wait": {"avg_ms": 82.022, "count": 23, "max_ms": 91.345, "p95_ms": 90.761}, "visualize": {"avg_ms": 3.675, "count": 23, "max_ms": 3.908, "p95_ms": 3.898}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.989, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 5082}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9471.512, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T09:17:50+09:00", "last_error_summary": "[h264 @ 0x562420808000] error while decoding MB 48 29, bytestream -19 [in#0/rtsp @ 0x5624207c0e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5624207c0e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 88, "h264_decode_error_count": 88, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:17:50+09:00", "last_error_summary": "[h264 @ 0x562420808000] error while decoding MB 48 29, bytestream -19 [in#0/rtsp @ 0x5624207c0e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5624207c0e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T09:17:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 549.927, "last_started_at": "2026-07-02T09:08:40+09:00", "pid": 52195, "process": {"exists": false, "pid": 52195}, "restart_count": 87, "restart_count_per_hour": 10, "rtsp_timeout_count": 158, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 88}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.626, "sample_count": 2414, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 609924, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:17:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T09:17:50+09:00", "last_frame_read_age_sec": 0.114, "last_frame_read_at": "2026-07-02T09:17:50+09:00", "mjpeg": {"age_sec": 0.11, "exists": true, "mtime": "2026-07-02T09:17:50.523598+09:00", "mtime_epoch": 1782951470.5235977, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 224567}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T09:17:50+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1153.719, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 23, "pushed_frames": 23, "read_frames": 22, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.769, "pushed": 0.769, "read": 0.735}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.667, "count": 23, "max_ms": 5.867, "p95_ms": 5.73}, "gstreamer_push": {"avg_ms": 1.653, "count": 23, "max_ms": 3.836, "p95_ms": 3.779}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.3, "count": 23, "max_ms": 68.22, "p95_ms": 61.243}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.29, "count": 23, "max_ms": 92.305, "p95_ms": 91.332}, "yolo11_640_fire_11class": {"avg_ms": 70.929, "count": 23, "max_ms": 83.294, "p95_ms": 82.674}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.599, "count": 23, "max_ms": 84.136, "p95_ms": 84.029}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.881, "count": 23, "max_ms": 106.704, "p95_ms": 105.741}, "read_interval": {"avg_ms": 1359.633, "count": 22, "max_ms": 17525.795, "p95_ms": 10164.075}, "triton_wait": {"avg_ms": 82.303, "count": 23, "max_ms": 92.317, "p95_ms": 91.345}, "visualize": {"avg_ms": 3.674, "count": 23, "max_ms": 3.908, "p95_ms": 3.898}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.922, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 5082}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9471.41, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 88, "h264_decode_error_count": 88, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:17:50+09:00", "last_error_summary": "[h264 @ 0x562420808000] error while decoding MB 48 29, bytestream -19 [in#0/rtsp @ 0x5624207c0e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5624207c0e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T09:17:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 549.934, "last_started_at": "2026-07-02T09:08:40+09:00", "pid": 52195, "process": {"exists": false, "pid": 52195}, "restart_count": 87, "restart_count_per_hour": 10, "rtsp_timeout_count": 158, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": false, "start_count": 88}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.626, "sample_count": 2414, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 609924, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:17:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-02T09:17:50+09:00", "last_frame_read_age_sec": 0.121, "last_frame_read_at": "2026-07-02T09:17:50+09:00", "mjpeg": {"age_sec": 0.117, "exists": true, "mtime": "2026-07-02T09:17:50.523598+09:00", "mtime_epoch": 1782951470.5235977, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 224567}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T09:17:50+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1153.719, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 23, "pushed_frames": 23, "read_frames": 22, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.768, "pushed": 0.768, "read": 0.735}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.667, "count": 23, "max_ms": 5.867, "p95_ms": 5.73}, "gstreamer_push": {"avg_ms": 1.653, "count": 23, "max_ms": 3.836, "p95_ms": 3.779}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.3, "count": 23, "max_ms": 68.22, "p95_ms": 61.243}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.29, "count": 23, "max_ms": 92.305, "p95_ms": 91.332}, "yolo11_640_fire_11class": {"avg_ms": 70.929, "count": 23, "max_ms": 83.294, "p95_ms": 82.674}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 73.599, "count": 23, "max_ms": 84.136, "p95_ms": 84.029}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.881, "count": 23, "max_ms": 106.704, "p95_ms": 105.741}, "read_interval": {"avg_ms": 1359.633, "count": 22, "max_ms": 17525.795, "p95_ms": 10164.075}, "triton_wait": {"avg_ms": 82.303, "count": 23, "max_ms": 92.317, "p95_ms": 91.345}, "visualize": {"avg_ms": 3.674, "count": 23, "max_ms": 3.908, "p95_ms": 3.898}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.929, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 5082}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9584.984, "limit_mb": 8796093022207.996}, "detail": {"pid": 52561, "restart_count": 88, "start_count": 89}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 88, "h264_decode_error_count": 88, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:17:50+09:00", "last_error_summary": "[h264 @ 0x562420808000] error while decoding MB 48 29, bytestream -19 [in#0/rtsp @ 0x5624207c0e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5624207c0e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T09:17:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T09:17:53+09:00", "pid": 52561, "process": {"exists": true, "fd_count": 5, "hwm_mb": 145.098, "name": "ffmpeg", "pid": 52561, "rss_mb": 145.098, "state": "R (running)", "threads": 49}, "restart_count": 88, "restart_count_per_hour": 11, "rtsp_timeout_count": 158, "rtsp_timeout_count_per_hour": 20, "rtsp_tolerant_mode": false, "running": true, "start_count": 89}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 106.634, "sample_count": 2377, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 609924, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:17:50+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.454, "last_frame_pushed_at": "2026-07-02T09:17:50+09:00", "last_frame_read_age_sec": 4.563, "last_frame_read_at": "2026-07-02T09:17:50+09:00", "mjpeg": {"age_sec": 4.435, "exists": true, "mtime": "2026-07-02T09:17:50.647601+09:00", "mtime_epoch": 1782951470.647601, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 224508}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T09:17:55+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1153.719, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.819, "pushed": 0.819, "read": 0.819}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.673, "count": 13, "max_ms": 5.867, "p95_ms": 5.867}, "gstreamer_push": {"avg_ms": 1.714, "count": 13, "max_ms": 3.836, "p95_ms": 3.836}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.431, "count": 13, "max_ms": 68.22, "p95_ms": 68.22}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.817, "count": 13, "max_ms": 92.305, "p95_ms": 92.305}, "yolo11_640_fire_11class": {"avg_ms": 64.224, "count": 13, "max_ms": 80.004, "p95_ms": 80.004}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.079, "count": 13, "max_ms": 82.34, "p95_ms": 82.34}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.322, "count": 13, "max_ms": 106.704, "p95_ms": 106.704}, "read_interval": {"avg_ms": 2217.615, "count": 13, "max_ms": 17525.795, "p95_ms": 17525.795}, "triton_wait": {"avg_ms": 76.83, "count": 13, "max_ms": 92.317, "p95_ms": 92.317}, "visualize": {"avg_ms": 3.65, "count": 13, "max_ms": 3.734, "p95_ms": 3.734}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 15.867, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 5089}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9476.949, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x5620e2c54200] missing picture in access unit with size 95 [h264 @ 0x5620e2c5ff80] no frame! [in#0/rtsp @ 0x5620e2c4fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5620e2c4fe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x5620e2c54840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 89, "h264_decode_error_count": 89, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:20:04+09:00", "last_error_summary": "[NULL @ 0x5620e2c54200] missing picture in access unit with size 95 [h264 @ 0x5620e2c5ff80] no frame! [in#0/rtsp @ 0x5620e2c4fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5620e2c4fe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x5620e2c54840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T09:17:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.911, "last_started_at": "2026-07-02T09:17:53+09:00", "pid": 52561, "process": {"exists": false, "pid": 52561}, "restart_count": 88, "restart_count_per_hour": 11, "rtsp_timeout_count": 160, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 89}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 162.967, "sample_count": 2079, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 610742, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:20:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-02T09:20:04+09:00", "last_frame_read_age_sec": 0.066, "last_frame_read_at": "2026-07-02T09:20:04+09:00", "mjpeg": {"age_sec": 0.058, "exists": true, "mtime": "2026-07-02T09:20:04.934866+09:00", "mtime_epoch": 1782951604.9348657, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 254857}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T09:20:04+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1152.844, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 80, "pushed_frames": 80, "read_frames": 80, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.67, "pushed": 2.67, "read": 2.67}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.353, "count": 79, "max_ms": 10.115, "p95_ms": 9.882}, "gstreamer_push": {"avg_ms": 1.399, "count": 80, "max_ms": 6.52, "p95_ms": 1.447}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.538, "count": 79, "max_ms": 68.33, "p95_ms": 64.391}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.161, "count": 79, "max_ms": 91.884, "p95_ms": 90.681}, "yolo11_640_fire_11class": {"avg_ms": 73.0, "count": 79, "max_ms": 84.921, "p95_ms": 83.639}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.985, "count": 79, "max_ms": 85.705, "p95_ms": 83.622}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.949, "count": 80, "max_ms": 171.877, "p95_ms": 149.168}, "read_interval": {"avg_ms": 375.534, "count": 80, "max_ms": 11642.486, "p95_ms": 180.643}, "triton_wait": {"avg_ms": 82.233, "count": 79, "max_ms": 91.896, "p95_ms": 90.694}, "visualize": {"avg_ms": 3.834, "count": 79, "max_ms": 5.162, "p95_ms": 5.105}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.963, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 5089}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9476.797, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T09:20:04+09:00", "last_error_summary": "[NULL @ 0x5620e2c54200] missing picture in access unit with size 95 [h264 @ 0x5620e2c5ff80] no frame! [in#0/rtsp @ 0x5620e2c4fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5620e2c4fe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x5620e2c54840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 89, "h264_decode_error_count": 89, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:20:04+09:00", "last_error_summary": "[NULL @ 0x5620e2c54200] missing picture in access unit with size 95 [h264 @ 0x5620e2c5ff80] no frame! [in#0/rtsp @ 0x5620e2c4fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5620e2c4fe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x5620e2c54840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T09:20:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.933, "last_started_at": "2026-07-02T09:17:53+09:00", "pid": 52561, "process": {"exists": false, "pid": 52561}, "restart_count": 88, "restart_count_per_hour": 11, "rtsp_timeout_count": 160, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 89}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 162.967, "sample_count": 2080, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 610743, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:20:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-02T09:20:05+09:00", "last_frame_read_age_sec": 0.088, "last_frame_read_at": "2026-07-02T09:20:04+09:00", "mjpeg": {"age_sec": 0.08, "exists": true, "mtime": "2026-07-02T09:20:04.934866+09:00", "mtime_epoch": 1782951604.9348657, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 254857}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T09:20:05+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1152.844, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 81, "pushed_frames": 81, "read_frames": 80, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.701, "pushed": 2.701, "read": 2.668}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.344, "count": 80, "max_ms": 10.115, "p95_ms": 9.822}, "gstreamer_push": {"avg_ms": 1.398, "count": 81, "max_ms": 6.52, "p95_ms": 1.447}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.481, "count": 80, "max_ms": 68.33, "p95_ms": 59.878}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.845, "count": 80, "max_ms": 91.884, "p95_ms": 90.615}, "yolo11_640_fire_11class": {"avg_ms": 72.767, "count": 80, "max_ms": 84.921, "p95_ms": 82.806}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.711, "count": 80, "max_ms": 85.705, "p95_ms": 82.96}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.58, "count": 81, "max_ms": 171.877, "p95_ms": 149.168}, "read_interval": {"avg_ms": 375.534, "count": 80, "max_ms": 11642.486, "p95_ms": 180.643}, "triton_wait": {"avg_ms": 81.917, "count": 80, "max_ms": 91.896, "p95_ms": 90.627}, "visualize": {"avg_ms": 3.861, "count": 80, "max_ms": 6.056, "p95_ms": 5.105}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.984, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 5089}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9476.637, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 89, "h264_decode_error_count": 89, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:20:04+09:00", "last_error_summary": "[NULL @ 0x5620e2c54200] missing picture in access unit with size 95 [h264 @ 0x5620e2c5ff80] no frame! [in#0/rtsp @ 0x5620e2c4fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5620e2c4fe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x5620e2c54840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T09:20:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.94, "last_started_at": "2026-07-02T09:17:53+09:00", "pid": 52561, "process": {"exists": false, "pid": 52561}, "restart_count": 88, "restart_count_per_hour": 11, "rtsp_timeout_count": 160, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": false, "start_count": 89}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 162.967, "sample_count": 2080, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 610743, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:20:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 0.017, "last_frame_pushed_at": "2026-07-02T09:20:05+09:00", "last_frame_read_age_sec": 0.095, "last_frame_read_at": "2026-07-02T09:20:04+09:00", "mjpeg": {"age_sec": 0.087, "exists": true, "mtime": "2026-07-02T09:20:04.934866+09:00", "mtime_epoch": 1782951604.9348657, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 254857}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T09:20:05+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1152.844, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 81, "pushed_frames": 81, "read_frames": 80, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.701, "pushed": 2.701, "read": 2.667}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.344, "count": 80, "max_ms": 10.115, "p95_ms": 9.822}, "gstreamer_push": {"avg_ms": 1.398, "count": 81, "max_ms": 6.52, "p95_ms": 1.447}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.481, "count": 80, "max_ms": 68.33, "p95_ms": 59.878}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 81.845, "count": 80, "max_ms": 91.884, "p95_ms": 90.615}, "yolo11_640_fire_11class": {"avg_ms": 72.767, "count": 80, "max_ms": 84.921, "p95_ms": 82.806}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.711, "count": 80, "max_ms": 85.705, "p95_ms": 82.96}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.58, "count": 81, "max_ms": 171.877, "p95_ms": 149.168}, "read_interval": {"avg_ms": 375.534, "count": 80, "max_ms": 11642.486, "p95_ms": 180.643}, "triton_wait": {"avg_ms": 81.917, "count": 80, "max_ms": 91.896, "p95_ms": 90.627}, "visualize": {"avg_ms": 3.861, "count": 80, "max_ms": 6.056, "p95_ms": 5.105}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260701_144823_d035afba.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 10, "tracking_summary": 5089}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 9497.41, "limit_mb": 8796093022207.996}, "detail": {"pid": 52713, "restart_count": 89, "start_count": 90}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 89, "h264_decode_error_count": 89, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:20:04+09:00", "last_error_summary": "[NULL @ 0x5620e2c54200] missing picture in access unit with size 95 [h264 @ 0x5620e2c5ff80] no frame! [in#0/rtsp @ 0x5620e2c4fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5620e2c4fe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x5620e2c54840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T09:20:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T09:20:07+09:00", "pid": 52713, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.27, "name": "ffmpeg", "pid": 52713, "rss_mb": 58.082, "state": "S (sleeping)", "threads": 1}, "restart_count": 89, "restart_count_per_hour": 12, "rtsp_timeout_count": 160, "rtsp_timeout_count_per_hour": 22, "rtsp_tolerant_mode": false, "running": true, "start_count": 90}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 163.059, "sample_count": 2041, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 610743, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:20:04+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "d035afba", "last_frame_pushed_age_sec": 4.75, "last_frame_pushed_at": "2026-07-02T09:20:05+09:00", "last_frame_read_age_sec": 4.828, "last_frame_read_at": "2026-07-02T09:20:04+09:00", "mjpeg": {"age_sec": 4.732, "exists": true, "mtime": "2026-07-02T09:20:05.022868+09:00", "mtime_epoch": 1782951605.022868, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 254758}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T09:20:09+09:00", "process": {"exists": true, "fd_count": 136, "hwm_mb": 1402.105, "name": "celery", "pid": 267, "rss_mb": 1152.844, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357%21@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 44, "pushed_frames": 44, "read_frames": 43, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.469, "pushed": 1.469, "read": 1.435}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.828, "count": 44, "max_ms": 6.595, "p95_ms": 6.263}, "gstreamer_push": {"avg_ms": 1.534, "count": 44, "max_ms": 6.52, "p95_ms": 3.429}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.065, "count": 44, "max_ms": 59.878, "p95_ms": 57.295}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 82.43, "count": 44, "max_ms": 91.884, "p95_ms": 90.758}, "yolo11_640_fire_11class": {"avg_ms": 73.339, "count": 44, "max_ms": 84.921, "p95_ms": 83.639}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.383, "count": 44, "max_ms": 85.705, "p95_ms": 84.657}}, "truncated_count": 0}, "processing_total": {"avg_ms": 96.962, "count": 44, "max_ms": 106.222, "p95_ms": 105.242}, "read_interval": {"avg_ms": 586.839, "count": 43, "max_ms": 11642.486, "p95_ms": 123.909}, "triton_wait": {"avg_ms": 82.443, "count": 44, "max_ms": 91.896, "p95_ms": 90.772}, "visualize": {"avg_ms": 3.7, "count": 44, "max_ms": 6.056, "p95_ms": 3.854}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.961, "window_s": 30.0}, "started_at": "2026-07-01T14:48:23+09:00", "state": "running", "task_id": "61a43aeb-00c9-418a-a5a3-dfb3b340f364", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 2748.832, "limit_mb": 8796093022207.996}, "detail": {"pid": 276, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T09:20:40+09:00", "pid": 276, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.527, "name": "ffmpeg", "pid": 276, "rss_mb": 141.527, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.904}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": [], "level": "warning", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["mjpeg_age_sec>10s"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 18.013, "exists": true, "mtime": "2026-07-02T09:20:24.323324+09:00", "mtime_epoch": 1782951624.3233242, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189572}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T09:20:42+09:00", "process": {"exists": true, "fd_count": 106, "hwm_mb": 467.824, "name": "celery", "pid": 237, "rss_mb": 467.824, "state": "R (running)", "threads": 6}, "reason_codes": ["mjpeg_output_stale_warning"], "recommended_action": "degrade_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": false, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 2748.832, "limit_mb": 8796093022207.996}, "detail": {"pid": 275, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T09:20:40+09:00", "pid": 275, "process": {"exists": true, "fd_count": 5, "hwm_mb": 202.027, "name": "ffmpeg", "pid": 275, "rss_mb": 202.027, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.904}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": [], "level": "warning", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["mjpeg_age_sec>10s"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 18.02, "exists": true, "mtime": "2026-07-02T09:20:24.315324+09:00", "mtime_epoch": 1782951624.315324, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 255014}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T09:20:42+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 467.863, "name": "celery", "pid": 267, "rss_mb": 467.863, "state": "R (running)", "threads": 6}, "reason_codes": ["mjpeg_output_stale_warning"], "recommended_action": "degrade_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": false, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/25/tracking_analysis_20260702_092319_8b1c350e.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "25", "cgroup_memory": {"current_mb": 3880.312, "limit_mb": 8796093022207.996}, "detail": {"pid": 612, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": [], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T09:23:19+09:00", "pid": 612, "process": {"exists": true, "fd_count": 5, "hwm_mb": 137.613, "name": "ffmpeg", "pid": 612, "rss_mb": 137.613, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.742}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "8b1c350e", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 64482.734, "exists": true, "mtime": "2026-07-01T15:28:38.632438+09:00", "mtime_epoch": 1782887318.6324377, "path": "/data/playlist/streaming/25/mjpeg.jpg", "size_bytes": 261957}, "monitor_url": "http://192.168.50.244:1223/mjpeg/25", "observed_at": "2026-07-02T09:23:21+09:00", "process": {"exists": true, "fd_count": 110, "hwm_mb": 471.254, "name": "celery", "pid": 239, "rss_mb": 471.254, "state": "R (running)", "threads": 7}, "reason_codes": ["mjpeg_output_stale_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "file:///data/box_steam/260423_seochosite_dataset/input/video/20260507_155115_260507_샘플다시저장_원본영상_sample_live_720p_10fps_aligned/260511_6분30초부터_8분15초까지_clip/clip_0390-0495s/2026-04-23 12_59_58 이동형 #1__live_camfix_1280x720_10fps_crf23__clip_0390-0495s.mp4", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-02T09:23:19+09:00", "state": "running", "task_id": "9ffad948-483d-45cf-b40e-dfc9e0a16e46", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 83}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 3825.199, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55ac34085000] error while decoding MB 1 64, bytestream -25 [in#0/rtsp @ 0x55ac33f7fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ac33f7fe00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:38:09+09:00", "last_error_summary": "[h264 @ 0x55ac34085000] error while decoding MB 1 64, bytestream -25 [in#0/rtsp @ 0x55ac33f7fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ac33f7fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 1048.757, "last_started_at": "2026-07-02T09:20:40+09:00", "pid": 275, "process": {"exists": false, "pid": 275}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 139.719, "sample_count": 2564, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 10039, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:38:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T09:38:09+09:00", "last_frame_read_age_sec": 0.069, "last_frame_read_at": "2026-07-02T09:38:09+09:00", "mjpeg": {"age_sec": 0.061, "exists": true, "mtime": "2026-07-02T09:38:09.029980+09:00", "mtime_epoch": 1782952689.0299802, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 246883}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T09:38:09+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1129.918, "name": "celery", "pid": 267, "rss_mb": 966.902, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.778, "pushed": 0.778, "read": 0.778}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.664, "count": 13, "max_ms": 1.696, "p95_ms": 1.696}, "gstreamer_push": {"avg_ms": 1.392, "count": 13, "max_ms": 1.477, "p95_ms": 1.477}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.019, "count": 13, "max_ms": 66.057, "p95_ms": 66.057}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.558, "count": 13, "max_ms": 78.107, "p95_ms": 78.107}, "yolo11_640_fire_11class": {"avg_ms": 53.355, "count": 13, "max_ms": 67.386, "p95_ms": 67.386}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.566, "count": 13, "max_ms": 75.307, "p95_ms": 75.307}}, "truncated_count": 0}, "processing_total": {"avg_ms": 67.014, "count": 13, "max_ms": 88.581, "p95_ms": 88.581}, "read_interval": {"avg_ms": 2480.775, "count": 13, "max_ms": 15775.234, "p95_ms": 15775.234}, "triton_wait": {"avg_ms": 56.542, "count": 13, "max_ms": 78.12, "p95_ms": 78.12}, "visualize": {"avg_ms": 3.675, "count": 13, "max_ms": 3.825, "p95_ms": 3.825}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 16.713, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 83}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 3825.156, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T09:38:09+09:00", "last_error_summary": "[h264 @ 0x55ac34085000] error while decoding MB 1 64, bytestream -25 [in#0/rtsp @ 0x55ac33f7fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ac33f7fe00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:38:09+09:00", "last_error_summary": "[h264 @ 0x55ac34085000] error while decoding MB 1 64, bytestream -25 [in#0/rtsp @ 0x55ac33f7fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ac33f7fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T09:38:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 1048.768, "last_started_at": "2026-07-02T09:20:40+09:00", "pid": 275, "process": {"exists": false, "pid": 275}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 139.719, "sample_count": 2564, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 10039, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:38:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-02T09:38:09+09:00", "last_frame_read_age_sec": 0.08, "last_frame_read_at": "2026-07-02T09:38:09+09:00", "mjpeg": {"age_sec": 0.072, "exists": true, "mtime": "2026-07-02T09:38:09.029980+09:00", "mtime_epoch": 1782952689.0299802, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 246883}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T09:38:09+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1129.918, "name": "celery", "pid": 267, "rss_mb": 966.945, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.777, "pushed": 0.777, "read": 0.777}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.664, "count": 13, "max_ms": 1.696, "p95_ms": 1.696}, "gstreamer_push": {"avg_ms": 1.392, "count": 13, "max_ms": 1.477, "p95_ms": 1.477}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.019, "count": 13, "max_ms": 66.057, "p95_ms": 66.057}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.558, "count": 13, "max_ms": 78.107, "p95_ms": 78.107}, "yolo11_640_fire_11class": {"avg_ms": 53.355, "count": 13, "max_ms": 67.386, "p95_ms": 67.386}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.566, "count": 13, "max_ms": 75.307, "p95_ms": 75.307}}, "truncated_count": 0}, "processing_total": {"avg_ms": 67.014, "count": 13, "max_ms": 88.581, "p95_ms": 88.581}, "read_interval": {"avg_ms": 2480.775, "count": 13, "max_ms": 15775.234, "p95_ms": 15775.234}, "triton_wait": {"avg_ms": 56.542, "count": 13, "max_ms": 78.12, "p95_ms": 78.12}, "visualize": {"avg_ms": 3.675, "count": 13, "max_ms": 3.825, "p95_ms": 3.825}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 16.722, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 83}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 3824.891, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:38:09+09:00", "last_error_summary": "[h264 @ 0x55ac34085000] error while decoding MB 1 64, bytestream -25 [in#0/rtsp @ 0x55ac33f7fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ac33f7fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T09:38:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 1048.776, "last_started_at": "2026-07-02T09:20:40+09:00", "pid": 275, "process": {"exists": false, "pid": 275}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 139.719, "sample_count": 2564, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 10039, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:38:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.023, "last_frame_pushed_at": "2026-07-02T09:38:09+09:00", "last_frame_read_age_sec": 0.088, "last_frame_read_at": "2026-07-02T09:38:09+09:00", "mjpeg": {"age_sec": 0.008, "exists": true, "mtime": "2026-07-02T09:38:09.101981+09:00", "mtime_epoch": 1782952689.1019814, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 246816}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T09:38:09+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1129.918, "name": "celery", "pid": 267, "rss_mb": 966.945, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.777, "pushed": 0.777, "read": 0.777}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.664, "count": 13, "max_ms": 1.696, "p95_ms": 1.696}, "gstreamer_push": {"avg_ms": 1.392, "count": 13, "max_ms": 1.477, "p95_ms": 1.477}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.019, "count": 13, "max_ms": 66.057, "p95_ms": 66.057}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.558, "count": 13, "max_ms": 78.107, "p95_ms": 78.107}, "yolo11_640_fire_11class": {"avg_ms": 53.355, "count": 13, "max_ms": 67.386, "p95_ms": 67.386}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.566, "count": 13, "max_ms": 75.307, "p95_ms": 75.307}}, "truncated_count": 0}, "processing_total": {"avg_ms": 67.014, "count": 13, "max_ms": 88.581, "p95_ms": 88.581}, "read_interval": {"avg_ms": 2480.775, "count": 13, "max_ms": 15775.234, "p95_ms": 15775.234}, "triton_wait": {"avg_ms": 56.542, "count": 13, "max_ms": 78.12, "p95_ms": 78.12}, "visualize": {"avg_ms": 3.675, "count": 13, "max_ms": 3.825, "p95_ms": 3.825}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 16.729, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 83}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 3943.457, "limit_mb": 8796093022207.996}, "detail": {"pid": 1105, "restart_count": 1, "start_count": 2}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:38:09+09:00", "last_error_summary": "[h264 @ 0x55ac34085000] error while decoding MB 1 64, bytestream -25 [in#0/rtsp @ 0x55ac33f7fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55ac33f7fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T09:38:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T09:38:11+09:00", "pid": 1105, "process": {"exists": true, "fd_count": 5, "hwm_mb": 156.715, "name": "ffmpeg", "pid": 1105, "rss_mb": 156.551, "state": "R (running)", "threads": 49}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 140.435, "sample_count": 2525, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 10039, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:38:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 4.128, "last_frame_pushed_at": "2026-07-02T09:38:09+09:00", "last_frame_read_age_sec": 4.193, "last_frame_read_at": "2026-07-02T09:38:09+09:00", "mjpeg": {"age_sec": 4.113, "exists": true, "mtime": "2026-07-02T09:38:09.101981+09:00", "mtime_epoch": 1782952689.1019814, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 246816}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T09:38:13+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1129.918, "name": "celery", "pid": 267, "rss_mb": 966.945, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.624, "pushed": 0.624, "read": 0.624}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.664, "count": 13, "max_ms": 1.696, "p95_ms": 1.696}, "gstreamer_push": {"avg_ms": 1.392, "count": 13, "max_ms": 1.477, "p95_ms": 1.477}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.019, "count": 13, "max_ms": 66.057, "p95_ms": 66.057}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.558, "count": 13, "max_ms": 78.107, "p95_ms": 78.107}, "yolo11_640_fire_11class": {"avg_ms": 53.355, "count": 13, "max_ms": 67.386, "p95_ms": 67.386}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.566, "count": 13, "max_ms": 75.307, "p95_ms": 75.307}}, "truncated_count": 0}, "processing_total": {"avg_ms": 67.014, "count": 13, "max_ms": 88.581, "p95_ms": 88.581}, "read_interval": {"avg_ms": 2480.775, "count": 13, "max_ms": 15775.234, "p95_ms": 15775.234}, "triton_wait": {"avg_ms": 56.542, "count": 13, "max_ms": 78.12, "p95_ms": 78.12}, "visualize": {"avg_ms": 3.675, "count": 13, "max_ms": 3.825, "p95_ms": 3.825}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 20.834, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 101}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 3830.531, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x561d7ed93000] error while decoding MB 44 31, bytestream -13 [in#0/rtsp @ 0x561d7ed4be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561d7ed4be00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:42:36+09:00", "last_error_summary": "[h264 @ 0x561d7ed93000] error while decoding MB 44 31, bytestream -13 [in#0/rtsp @ 0x561d7ed4be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561d7ed4be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T09:38:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 265.545, "last_started_at": "2026-07-02T09:38:11+09:00", "pid": 1105, "process": {"exists": false, "pid": 1105}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.822, "sample_count": 2202, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12228, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:42:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.071, "last_frame_pushed_at": "2026-07-02T09:42:36+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-02T09:42:36+09:00", "mjpeg": {"age_sec": 0.054, "exists": true, "mtime": "2026-07-02T09:42:36.704392+09:00", "mtime_epoch": 1782952956.7043917, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 236198}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T09:42:36+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1129.918, "name": "celery", "pid": 267, "rss_mb": 967.977, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.704, "pushed": 0.704, "read": 0.762}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.871, "count": 13, "max_ms": 2.438, "p95_ms": 2.438}, "gstreamer_push": {"avg_ms": 1.335, "count": 12, "max_ms": 1.471, "p95_ms": 1.471}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.045, "count": 13, "max_ms": 48.835, "p95_ms": 48.835}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.871, "count": 13, "max_ms": 51.797, "p95_ms": 51.797}, "yolo11_640_fire_11class": {"avg_ms": 49.629, "count": 13, "max_ms": 51.785, "p95_ms": 51.785}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 47.576, "count": 13, "max_ms": 49.425, "p95_ms": 49.425}}, "truncated_count": 0}, "processing_total": {"avg_ms": 60.456, "count": 12, "max_ms": 62.479, "p95_ms": 62.479}, "read_interval": {"avg_ms": 2480.627, "count": 13, "max_ms": 16177.284, "p95_ms": 16177.284}, "triton_wait": {"avg_ms": 50.176, "count": 13, "max_ms": 51.807, "p95_ms": 51.807}, "visualize": {"avg_ms": 3.673, "count": 13, "max_ms": 4.093, "p95_ms": 4.093}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.053, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 101}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 3830.398, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T09:42:36+09:00", "last_error_summary": "[h264 @ 0x561d7ed93000] error while decoding MB 44 31, bytestream -13 [in#0/rtsp @ 0x561d7ed4be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561d7ed4be00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:42:36+09:00", "last_error_summary": "[h264 @ 0x561d7ed93000] error while decoding MB 44 31, bytestream -13 [in#0/rtsp @ 0x561d7ed4be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561d7ed4be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T09:42:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 265.561, "last_started_at": "2026-07-02T09:38:11+09:00", "pid": 1105, "process": {"exists": false, "pid": 1105}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.822, "sample_count": 2203, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12228, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:42:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T09:42:36+09:00", "last_frame_read_age_sec": 0.077, "last_frame_read_at": "2026-07-02T09:42:36+09:00", "mjpeg": {"age_sec": 0.07, "exists": true, "mtime": "2026-07-02T09:42:36.704392+09:00", "mtime_epoch": 1782952956.7043917, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 236198}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T09:42:36+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1129.918, "name": "celery", "pid": 267, "rss_mb": 967.98, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.762, "pushed": 0.762, "read": 0.762}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.871, "count": 13, "max_ms": 2.438, "p95_ms": 2.438}, "gstreamer_push": {"avg_ms": 1.336, "count": 13, "max_ms": 1.471, "p95_ms": 1.471}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.045, "count": 13, "max_ms": 48.835, "p95_ms": 48.835}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.871, "count": 13, "max_ms": 51.797, "p95_ms": 51.797}, "yolo11_640_fire_11class": {"avg_ms": 49.629, "count": 13, "max_ms": 51.785, "p95_ms": 51.785}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 47.576, "count": 13, "max_ms": 49.425, "p95_ms": 49.425}}, "truncated_count": 0}, "processing_total": {"avg_ms": 60.893, "count": 13, "max_ms": 66.138, "p95_ms": 66.138}, "read_interval": {"avg_ms": 2480.627, "count": 13, "max_ms": 16177.284, "p95_ms": 16177.284}, "triton_wait": {"avg_ms": 50.176, "count": 13, "max_ms": 51.807, "p95_ms": 51.807}, "visualize": {"avg_ms": 3.673, "count": 13, "max_ms": 4.093, "p95_ms": 4.093}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.069, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 101}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 3830.391, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:42:36+09:00", "last_error_summary": "[h264 @ 0x561d7ed93000] error while decoding MB 44 31, bytestream -13 [in#0/rtsp @ 0x561d7ed4be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561d7ed4be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T09:42:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 265.568, "last_started_at": "2026-07-02T09:38:11+09:00", "pid": 1105, "process": {"exists": false, "pid": 1105}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.822, "sample_count": 2203, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12228, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:42:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-02T09:42:36+09:00", "last_frame_read_age_sec": 0.084, "last_frame_read_at": "2026-07-02T09:42:36+09:00", "mjpeg": {"age_sec": 0.077, "exists": true, "mtime": "2026-07-02T09:42:36.704392+09:00", "mtime_epoch": 1782952956.7043917, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 236198}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T09:42:36+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1129.918, "name": "celery", "pid": 267, "rss_mb": 967.98, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.761, "pushed": 0.761, "read": 0.761}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.871, "count": 13, "max_ms": 2.438, "p95_ms": 2.438}, "gstreamer_push": {"avg_ms": 1.336, "count": 13, "max_ms": 1.471, "p95_ms": 1.471}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.045, "count": 13, "max_ms": 48.835, "p95_ms": 48.835}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.871, "count": 13, "max_ms": 51.797, "p95_ms": 51.797}, "yolo11_640_fire_11class": {"avg_ms": 49.629, "count": 13, "max_ms": 51.785, "p95_ms": 51.785}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 47.576, "count": 13, "max_ms": 49.425, "p95_ms": 49.425}}, "truncated_count": 0}, "processing_total": {"avg_ms": 60.893, "count": 13, "max_ms": 66.138, "p95_ms": 66.138}, "read_interval": {"avg_ms": 2480.627, "count": 13, "max_ms": 16177.284, "p95_ms": 16177.284}, "triton_wait": {"avg_ms": 50.176, "count": 13, "max_ms": 51.807, "p95_ms": 51.807}, "visualize": {"avg_ms": 3.673, "count": 13, "max_ms": 4.093, "p95_ms": 4.093}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.076, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 5, "tracking_summary": 101}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 3853.102, "limit_mb": 8796093022207.996}, "detail": {"pid": 1257, "restart_count": 2, "start_count": 3}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:42:36+09:00", "last_error_summary": "[h264 @ 0x561d7ed93000] error while decoding MB 44 31, bytestream -13 [in#0/rtsp @ 0x561d7ed4be00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x561d7ed4be00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T09:42:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T09:42:38+09:00", "pid": 1257, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.18, "name": "ffmpeg", "pid": 1257, "rss_mb": 58.18, "state": "S (sleeping)", "threads": 1}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 153.822, "sample_count": 2202, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12228, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:42:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 4.167, "last_frame_pushed_at": "2026-07-02T09:42:36+09:00", "last_frame_read_age_sec": 4.237, "last_frame_read_at": "2026-07-02T09:42:36+09:00", "mjpeg": {"age_sec": 4.154, "exists": true, "mtime": "2026-07-02T09:42:36.780389+09:00", "mtime_epoch": 1782952956.7803886, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 236201}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T09:42:40+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1129.918, "name": "celery", "pid": 267, "rss_mb": 967.98, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.612, "pushed": 0.612, "read": 0.612}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.871, "count": 13, "max_ms": 2.438, "p95_ms": 2.438}, "gstreamer_push": {"avg_ms": 1.336, "count": 13, "max_ms": 1.471, "p95_ms": 1.471}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.045, "count": 13, "max_ms": 48.835, "p95_ms": 48.835}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.871, "count": 13, "max_ms": 51.797, "p95_ms": 51.797}, "yolo11_640_fire_11class": {"avg_ms": 49.629, "count": 13, "max_ms": 51.785, "p95_ms": 51.785}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 47.576, "count": 13, "max_ms": 49.425, "p95_ms": 49.425}}, "truncated_count": 0}, "processing_total": {"avg_ms": 60.893, "count": 13, "max_ms": 66.138, "p95_ms": 66.138}, "read_interval": {"avg_ms": 2480.627, "count": 13, "max_ms": 16177.284, "p95_ms": 16177.284}, "triton_wait": {"avg_ms": 50.176, "count": 13, "max_ms": 51.807, "p95_ms": 51.807}, "visualize": {"avg_ms": 3.673, "count": 13, "max_ms": 4.093, "p95_ms": 4.093}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.229, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 107}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3860.488, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x562846e86f80] error while decoding MB 25 34, bytestream -19", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:43:13+09:00", "last_error_summary": "[h264 @ 0x562846e86f80] error while decoding MB 25 34, bytestream -19", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 1352.885, "last_started_at": "2026-07-02T09:20:40+09:00", "pid": 276, "process": {"exists": false, "pid": 276}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 89.039, "sample_count": 2419, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12930, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:43:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-02T09:43:13+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-02T09:43:13+09:00", "mjpeg": {"age_sec": 0.033, "exists": true, "mtime": "2026-07-02T09:43:13.187022+09:00", "mtime_epoch": 1782952993.1870217, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182634}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T09:43:13+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 868.734, "name": "celery", "pid": 237, "rss_mb": 863.832, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 117, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.903, "pushed": 3.903, "read": 3.903}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.253, "count": 117, "max_ms": 1.599, "p95_ms": 1.296}, "gstreamer_push": {"avg_ms": 1.518, "count": 117, "max_ms": 4.73, "p95_ms": 3.664}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.344, "count": 117, "max_ms": 70.014, "p95_ms": 65.893}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.458, "count": 117, "max_ms": 85.841, "p95_ms": 74.633}, "yolo11_640_fire_11class": {"avg_ms": 53.672, "count": 117, "max_ms": 71.421, "p95_ms": 68.52}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.523, "count": 117, "max_ms": 75.399, "p95_ms": 74.975}}, "truncated_count": 0}, "processing_total": {"avg_ms": 71.634, "count": 117, "max_ms": 96.117, "p95_ms": 85.462}, "read_interval": {"avg_ms": 256.546, "count": 117, "max_ms": 7705.419, "p95_ms": 169.974}, "triton_wait": {"avg_ms": 61.491, "count": 117, "max_ms": 85.852, "p95_ms": 75.25}, "visualize": {"avg_ms": 3.755, "count": 117, "max_ms": 5.082, "p95_ms": 4.25}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.981, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 107}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3860.137, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T09:43:13+09:00", "last_error_summary": "[h264 @ 0x562846e86f80] error while decoding MB 25 34, bytestream -19", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:43:13+09:00", "last_error_summary": "[h264 @ 0x562846e86f80] error while decoding MB 25 34, bytestream -19", "last_exit_at": "2026-07-02T09:43:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 1352.912, "last_started_at": "2026-07-02T09:20:40+09:00", "pid": 276, "process": {"exists": false, "pid": 276}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 89.039, "sample_count": 2419, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12931, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:43:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T09:43:13+09:00", "last_frame_read_age_sec": 0.076, "last_frame_read_at": "2026-07-02T09:43:13+09:00", "mjpeg": {"age_sec": 0.06, "exists": true, "mtime": "2026-07-02T09:43:13.187022+09:00", "mtime_epoch": 1782952993.1870217, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182634}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T09:43:13+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 868.734, "name": "celery", "pid": 237, "rss_mb": 863.891, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 117, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.901, "pushed": 3.901, "read": 3.901}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.256, "count": 117, "max_ms": 1.599, "p95_ms": 1.3}, "gstreamer_push": {"avg_ms": 1.518, "count": 117, "max_ms": 4.73, "p95_ms": 3.664}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.226, "count": 117, "max_ms": 70.014, "p95_ms": 65.893}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.403, "count": 117, "max_ms": 85.841, "p95_ms": 74.633}, "yolo11_640_fire_11class": {"avg_ms": 53.525, "count": 117, "max_ms": 71.421, "p95_ms": 68.52}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.338, "count": 117, "max_ms": 75.399, "p95_ms": 74.975}}, "truncated_count": 0}, "processing_total": {"avg_ms": 71.491, "count": 117, "max_ms": 96.117, "p95_ms": 85.462}, "read_interval": {"avg_ms": 256.546, "count": 117, "max_ms": 7705.419, "p95_ms": 169.974}, "triton_wait": {"avg_ms": 61.327, "count": 117, "max_ms": 85.852, "p95_ms": 75.25}, "visualize": {"avg_ms": 3.774, "count": 117, "max_ms": 5.782, "p95_ms": 4.355}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 107}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3860.129, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:43:13+09:00", "last_error_summary": "[h264 @ 0x562846e86f80] error while decoding MB 25 34, bytestream -19", "last_exit_at": "2026-07-02T09:43:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 1352.92, "last_started_at": "2026-07-02T09:20:40+09:00", "pid": 276, "process": {"exists": false, "pid": 276}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 89.039, "sample_count": 2419, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12931, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:43:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-02T09:43:13+09:00", "last_frame_read_age_sec": 0.084, "last_frame_read_at": "2026-07-02T09:43:13+09:00", "mjpeg": {"age_sec": 0.067, "exists": true, "mtime": "2026-07-02T09:43:13.187022+09:00", "mtime_epoch": 1782952993.1870217, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 182634}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T09:43:13+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 868.734, "name": "celery", "pid": 237, "rss_mb": 863.891, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 116, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.91, "pushed": 3.91, "read": 3.877}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.256, "count": 117, "max_ms": 1.599, "p95_ms": 1.3}, "gstreamer_push": {"avg_ms": 1.518, "count": 117, "max_ms": 4.73, "p95_ms": 3.664}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.226, "count": 117, "max_ms": 70.014, "p95_ms": 65.893}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.403, "count": 117, "max_ms": 85.841, "p95_ms": 74.633}, "yolo11_640_fire_11class": {"avg_ms": 53.525, "count": 117, "max_ms": 71.421, "p95_ms": 68.52}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.338, "count": 117, "max_ms": 75.399, "p95_ms": 74.975}}, "truncated_count": 0}, "processing_total": {"avg_ms": 71.491, "count": 117, "max_ms": 96.117, "p95_ms": 85.462}, "read_interval": {"avg_ms": 257.924, "count": 116, "max_ms": 7705.419, "p95_ms": 169.974}, "triton_wait": {"avg_ms": 61.327, "count": 117, "max_ms": 85.852, "p95_ms": 75.25}, "visualize": {"avg_ms": 3.774, "count": 117, "max_ms": 5.782, "p95_ms": 4.355}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.922, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 107}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3867.477, "limit_mb": 8796093022207.996}, "detail": {"pid": 1409, "restart_count": 1, "start_count": 2}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 1, "h264_decode_error_count": 1, "h264_decode_error_count_per_hour": 1, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:43:13+09:00", "last_error_summary": "[h264 @ 0x562846e86f80] error while decoding MB 25 34, bytestream -19", "last_exit_at": "2026-07-02T09:43:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T09:43:16+09:00", "pid": 1409, "process": {"exists": true, "fd_count": 4, "hwm_mb": 52.879, "name": "ffmpeg", "pid": 1409, "rss_mb": 52.879, "state": "S (sleeping)", "threads": 1}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 89.039, "sample_count": 2369, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 12931, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:43:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 5.001, "last_frame_pushed_at": "2026-07-02T09:43:13+09:00", "last_frame_read_age_sec": 5.071, "last_frame_read_at": "2026-07-02T09:43:13+09:00", "mjpeg": {"age_sec": 4.979, "exists": true, "mtime": "2026-07-02T09:43:13.263019+09:00", "mtime_epoch": 1782952993.2630188, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181822}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T09:43:18+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 868.734, "name": "celery", "pid": 237, "rss_mb": 863.891, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 89, "pushed_frames": 89, "read_frames": 89, "select_timeouts": 15, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.13, "pushed": 3.13, "read": 3.13}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.255, "count": 89, "max_ms": 1.561, "p95_ms": 1.296}, "gstreamer_push": {"avg_ms": 1.5, "count": 89, "max_ms": 4.73, "p95_ms": 1.624}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.392, "count": 89, "max_ms": 70.014, "p95_ms": 66.352}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.197, "count": 89, "max_ms": 85.841, "p95_ms": 75.074}, "yolo11_640_fire_11class": {"avg_ms": 54.856, "count": 89, "max_ms": 71.421, "p95_ms": 68.383}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.743, "count": 89, "max_ms": 75.399, "p95_ms": 75.039}}, "truncated_count": 0}, "processing_total": {"avg_ms": 72.678, "count": 89, "max_ms": 96.117, "p95_ms": 85.634}, "read_interval": {"avg_ms": 308.392, "count": 89, "max_ms": 7705.419, "p95_ms": 108.318}, "triton_wait": {"avg_ms": 62.514, "count": 89, "max_ms": 85.852, "p95_ms": 75.411}, "visualize": {"avg_ms": 3.791, "count": 89, "max_ms": 5.782, "p95_ms": 4.523}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 28.438, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 110}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3824.32, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56074dc21f40] error while decoding MB 9 27, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:44:46+09:00", "last_error_summary": "[h264 @ 0x56074dc21f40] error while decoding MB 9 27, bytestream -7", "last_exit_at": "2026-07-02T09:43:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 89.827, "last_started_at": "2026-07-02T09:43:16+09:00", "pid": 1409, "process": {"exists": false, "pid": 1409}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 88.275, "sample_count": 1814, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 13254, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:44:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-02T09:44:46+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-02T09:44:46+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-02T09:44:46.020051+09:00", "mtime_epoch": 1782953086.0200505, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179610}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T09:44:46+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 868.734, "name": "celery", "pid": 237, "rss_mb": 864.152, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 145, "pushed_frames": 145, "read_frames": 145, "select_timeouts": 13, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.833, "pushed": 4.833, "read": 4.833}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.256, "count": 144, "max_ms": 1.772, "p95_ms": 1.344}, "gstreamer_push": {"avg_ms": 1.467, "count": 145, "max_ms": 3.963, "p95_ms": 1.673}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.769, "count": 144, "max_ms": 67.966, "p95_ms": 63.993}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.057, "count": 144, "max_ms": 78.986, "p95_ms": 75.082}, "yolo11_640_fire_11class": {"avg_ms": 52.062, "count": 144, "max_ms": 71.709, "p95_ms": 65.674}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.7, "count": 144, "max_ms": 76.827, "p95_ms": 73.844}}, "truncated_count": 0}, "processing_total": {"avg_ms": 70.297, "count": 145, "max_ms": 90.42, "p95_ms": 87.151}, "read_interval": {"avg_ms": 207.147, "count": 145, "max_ms": 5602.441, "p95_ms": 430.662}, "triton_wait": {"avg_ms": 60.084, "count": 144, "max_ms": 78.998, "p95_ms": 75.591}, "visualize": {"avg_ms": 3.742, "count": 144, "max_ms": 6.388, "p95_ms": 4.014}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 110}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3824.023, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T09:44:46+09:00", "last_error_summary": "[h264 @ 0x56074dc21f40] error while decoding MB 9 27, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:44:46+09:00", "last_error_summary": "[h264 @ 0x56074dc21f40] error while decoding MB 9 27, bytestream -7", "last_exit_at": "2026-07-02T09:44:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 89.866, "last_started_at": "2026-07-02T09:43:16+09:00", "pid": 1409, "process": {"exists": false, "pid": 1409}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 88.275, "sample_count": 1815, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 13255, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:44:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T09:44:46+09:00", "last_frame_read_age_sec": 0.089, "last_frame_read_at": "2026-07-02T09:44:46+09:00", "mjpeg": {"age_sec": 0.087, "exists": true, "mtime": "2026-07-02T09:44:46.020051+09:00", "mtime_epoch": 1782953086.0200505, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179610}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T09:44:46+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 868.734, "name": "celery", "pid": 237, "rss_mb": 864.152, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 145, "pushed_frames": 145, "read_frames": 145, "select_timeouts": 13, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.842, "pushed": 4.842, "read": 4.842}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.255, "count": 145, "max_ms": 1.772, "p95_ms": 1.344}, "gstreamer_push": {"avg_ms": 1.459, "count": 145, "max_ms": 3.963, "p95_ms": 1.577}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.838, "count": 145, "max_ms": 67.966, "p95_ms": 63.993}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.077, "count": 145, "max_ms": 78.986, "p95_ms": 75.082}, "yolo11_640_fire_11class": {"avg_ms": 52.136, "count": 145, "max_ms": 71.709, "p95_ms": 65.674}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.815, "count": 145, "max_ms": 76.827, "p95_ms": 73.844}}, "truncated_count": 0}, "processing_total": {"avg_ms": 70.257, "count": 145, "max_ms": 90.42, "p95_ms": 86.676}, "read_interval": {"avg_ms": 207.147, "count": 145, "max_ms": 5602.441, "p95_ms": 430.662}, "triton_wait": {"avg_ms": 60.163, "count": 145, "max_ms": 78.998, "p95_ms": 75.591}, "visualize": {"avg_ms": 3.742, "count": 145, "max_ms": 6.388, "p95_ms": 4.014}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.947, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 110}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3823.793, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:44:46+09:00", "last_error_summary": "[h264 @ 0x56074dc21f40] error while decoding MB 9 27, bytestream -7", "last_exit_at": "2026-07-02T09:44:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 89.874, "last_started_at": "2026-07-02T09:43:16+09:00", "pid": 1409, "process": {"exists": false, "pid": 1409}, "restart_count": 1, "restart_count_per_hour": 1, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 2}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 88.275, "sample_count": 1815, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 13255, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:44:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-02T09:44:46+09:00", "last_frame_read_age_sec": 0.097, "last_frame_read_at": "2026-07-02T09:44:46+09:00", "mjpeg": {"age_sec": 0.094, "exists": true, "mtime": "2026-07-02T09:44:46.020051+09:00", "mtime_epoch": 1782953086.0200505, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 179610}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T09:44:46+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 868.734, "name": "celery", "pid": 237, "rss_mb": 864.152, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 145, "pushed_frames": 145, "read_frames": 145, "select_timeouts": 13, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.841, "pushed": 4.841, "read": 4.841}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.255, "count": 145, "max_ms": 1.772, "p95_ms": 1.344}, "gstreamer_push": {"avg_ms": 1.459, "count": 145, "max_ms": 3.963, "p95_ms": 1.577}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.838, "count": 145, "max_ms": 67.966, "p95_ms": 63.993}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.077, "count": 145, "max_ms": 78.986, "p95_ms": 75.082}, "yolo11_640_fire_11class": {"avg_ms": 52.136, "count": 145, "max_ms": 71.709, "p95_ms": 65.674}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.815, "count": 145, "max_ms": 76.827, "p95_ms": 73.844}}, "truncated_count": 0}, "processing_total": {"avg_ms": 70.257, "count": 145, "max_ms": 90.42, "p95_ms": 86.676}, "read_interval": {"avg_ms": 207.147, "count": 145, "max_ms": 5602.441, "p95_ms": 430.662}, "triton_wait": {"avg_ms": 60.163, "count": 145, "max_ms": 78.998, "p95_ms": 75.591}, "visualize": {"avg_ms": 3.742, "count": 145, "max_ms": 6.388, "p95_ms": 4.014}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.955, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 110}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3921.449, "limit_mb": 8796093022207.996}, "detail": {"pid": 1516, "restart_count": 2, "start_count": 3}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:44:46+09:00", "last_error_summary": "[h264 @ 0x56074dc21f40] error while decoding MB 9 27, bytestream -7", "last_exit_at": "2026-07-02T09:44:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T09:44:48+09:00", "pid": 1516, "process": {"exists": true, "fd_count": 5, "hwm_mb": 124.176, "name": "ffmpeg", "pid": 1516, "rss_mb": 124.176, "state": "S (sleeping)", "threads": 106}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 88.243, "sample_count": 1773, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 13255, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:44:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.161, "last_frame_pushed_at": "2026-07-02T09:44:46+09:00", "last_frame_read_age_sec": 4.245, "last_frame_read_at": "2026-07-02T09:44:46+09:00", "mjpeg": {"age_sec": 4.146, "exists": true, "mtime": "2026-07-02T09:44:46.116048+09:00", "mtime_epoch": 1782953086.1160479, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 180033}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T09:44:50+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 868.734, "name": "celery", "pid": 237, "rss_mb": 864.152, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 106, "pushed_frames": 106, "read_frames": 106, "select_timeouts": 13, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.921, "pushed": 3.921, "read": 3.921}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.235, "count": 106, "max_ms": 1.57, "p95_ms": 1.29}, "gstreamer_push": {"avg_ms": 1.447, "count": 106, "max_ms": 3.963, "p95_ms": 1.5}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.299, "count": 106, "max_ms": 67.966, "p95_ms": 64.177}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.731, "count": 106, "max_ms": 78.986, "p95_ms": 77.294}, "yolo11_640_fire_11class": {"avg_ms": 53.032, "count": 106, "max_ms": 71.709, "p95_ms": 67.191}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.01, "count": 106, "max_ms": 76.827, "p95_ms": 74.26}}, "truncated_count": 0}, "processing_total": {"avg_ms": 70.524, "count": 106, "max_ms": 90.028, "p95_ms": 87.405}, "read_interval": {"avg_ms": 251.085, "count": 106, "max_ms": 5602.441, "p95_ms": 589.033}, "triton_wait": {"avg_ms": 60.556, "count": 106, "max_ms": 78.998, "p95_ms": 77.306}, "visualize": {"avg_ms": 3.688, "count": 106, "max_ms": 4.465, "p95_ms": 3.884}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.035, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 114}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3914.734, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T09:44:46+09:00", "last_error_summary": "[h264 @ 0x56074dc21f40] error while decoding MB 9 27, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:44:46+09:00", "last_error_summary": "[h264 @ 0x56074dc21f40] error while decoding MB 9 27, bytestream -7", "last_exit_at": "2026-07-02T09:46:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 97.252, "last_started_at": "2026-07-02T09:44:48+09:00", "pid": 1516, "process": {"exists": false, "pid": 1516}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 109.919, "sample_count": 1271, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 13705, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:46:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T09:46:25+09:00", "last_frame_read_age_sec": 0.072, "last_frame_read_at": "2026-07-02T09:46:25+09:00", "mjpeg": {"age_sec": 0.067, "exists": true, "mtime": "2026-07-02T09:46:25.445576+09:00", "mtime_epoch": 1782953185.4455762, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190875}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T09:46:25+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 868.734, "name": "celery", "pid": 237, "rss_mb": 863.75, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 182, "pushed_frames": 182, "read_frames": 182, "select_timeouts": 11, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.147, "pushed": 6.147, "read": 6.147}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.247, "count": 182, "max_ms": 1.593, "p95_ms": 1.322}, "gstreamer_push": {"avg_ms": 1.715, "count": 182, "max_ms": 6.963, "p95_ms": 3.958}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.639, "count": 182, "max_ms": 69.996, "p95_ms": 64.802}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.788, "count": 182, "max_ms": 81.265, "p95_ms": 76.318}, "yolo11_640_fire_11class": {"avg_ms": 52.068, "count": 182, "max_ms": 70.407, "p95_ms": 67.151}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.346, "count": 182, "max_ms": 76.286, "p95_ms": 74.731}}, "truncated_count": 0}, "processing_total": {"avg_ms": 69.718, "count": 182, "max_ms": 91.578, "p95_ms": 86.586}, "read_interval": {"avg_ms": 166.931, "count": 182, "max_ms": 3654.389, "p95_ms": 229.134}, "triton_wait": {"avg_ms": 59.588, "count": 182, "max_ms": 81.277, "p95_ms": 76.33}, "visualize": {"avg_ms": 3.721, "count": 182, "max_ms": 4.358, "p95_ms": 3.9}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.61, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 114}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3914.703, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:44:46+09:00", "last_error_summary": "[h264 @ 0x56074dc21f40] error while decoding MB 9 27, bytestream -7", "last_exit_at": "2026-07-02T09:46:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 97.259, "last_started_at": "2026-07-02T09:44:48+09:00", "pid": 1516, "process": {"exists": false, "pid": 1516}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 109.919, "sample_count": 1271, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 13705, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:46:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-02T09:46:25+09:00", "last_frame_read_age_sec": 0.08, "last_frame_read_at": "2026-07-02T09:46:25+09:00", "mjpeg": {"age_sec": 0.075, "exists": true, "mtime": "2026-07-02T09:46:25.445576+09:00", "mtime_epoch": 1782953185.4455762, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190875}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T09:46:25+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 868.734, "name": "celery", "pid": 237, "rss_mb": 863.75, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 182, "pushed_frames": 182, "read_frames": 182, "select_timeouts": 11, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.145, "pushed": 6.145, "read": 6.145}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.247, "count": 182, "max_ms": 1.593, "p95_ms": 1.322}, "gstreamer_push": {"avg_ms": 1.715, "count": 182, "max_ms": 6.963, "p95_ms": 3.958}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.639, "count": 182, "max_ms": 69.996, "p95_ms": 64.802}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.788, "count": 182, "max_ms": 81.265, "p95_ms": 76.318}, "yolo11_640_fire_11class": {"avg_ms": 52.068, "count": 182, "max_ms": 70.407, "p95_ms": 67.151}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.346, "count": 182, "max_ms": 76.286, "p95_ms": 74.731}}, "truncated_count": 0}, "processing_total": {"avg_ms": 69.718, "count": 182, "max_ms": 91.578, "p95_ms": 86.586}, "read_interval": {"avg_ms": 166.931, "count": 182, "max_ms": 3654.389, "p95_ms": 229.134}, "triton_wait": {"avg_ms": 59.588, "count": 182, "max_ms": 81.277, "p95_ms": 76.33}, "visualize": {"avg_ms": 3.721, "count": 182, "max_ms": 4.358, "p95_ms": 3.9}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.617, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 114}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4013.711, "limit_mb": 8796093022207.996}, "detail": {"pid": 1623, "restart_count": 3, "start_count": 4}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 2, "h264_decode_error_count": 2, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:44:46+09:00", "last_error_summary": "[h264 @ 0x56074dc21f40] error while decoding MB 9 27, bytestream -7", "last_exit_at": "2026-07-02T09:46:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T09:46:27+09:00", "pid": 1623, "process": {"exists": true, "fd_count": 5, "hwm_mb": 131.082, "name": "ffmpeg", "pid": 1623, "rss_mb": 131.082, "state": "S (sleeping)", "threads": 106}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 110.413, "sample_count": 1228, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 13705, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:46:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.361, "last_frame_pushed_at": "2026-07-02T09:46:25+09:00", "last_frame_read_age_sec": 4.427, "last_frame_read_at": "2026-07-02T09:46:25+09:00", "mjpeg": {"age_sec": 4.346, "exists": true, "mtime": "2026-07-02T09:46:25.521574+09:00", "mtime_epoch": 1782953185.5215745, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191000}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T09:46:29+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 868.734, "name": "celery", "pid": 237, "rss_mb": 863.75, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 147, "pushed_frames": 147, "read_frames": 147, "select_timeouts": 11, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.335, "pushed": 5.335, "read": 5.335}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.248, "count": 147, "max_ms": 1.593, "p95_ms": 1.332}, "gstreamer_push": {"avg_ms": 1.715, "count": 147, "max_ms": 6.963, "p95_ms": 3.997}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.939, "count": 147, "max_ms": 69.996, "p95_ms": 65.224}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.697, "count": 147, "max_ms": 81.265, "p95_ms": 76.757}, "yolo11_640_fire_11class": {"avg_ms": 53.587, "count": 147, "max_ms": 70.407, "p95_ms": 67.157}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.356, "count": 147, "max_ms": 76.286, "p95_ms": 75.1}}, "truncated_count": 0}, "processing_total": {"avg_ms": 71.317, "count": 147, "max_ms": 91.578, "p95_ms": 86.991}, "read_interval": {"avg_ms": 178.468, "count": 147, "max_ms": 3654.389, "p95_ms": 161.145}, "triton_wait": {"avg_ms": 61.161, "count": 147, "max_ms": 81.277, "p95_ms": 76.769}, "visualize": {"avg_ms": 3.727, "count": 147, "max_ms": 4.358, "p95_ms": 3.905}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.556, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 159}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3923.105, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55c02a1a9b00] error while decoding MB 29 36, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:56:18+09:00", "last_error_summary": "[h264 @ 0x55c02a1a9b00] error while decoding MB 29 36, bytestream -5", "last_exit_at": "2026-07-02T09:46:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 590.68, "last_started_at": "2026-07-02T09:46:27+09:00", "pid": 1623, "process": {"exists": false, "pid": 1623}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 90.436, "sample_count": 2560, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 19090, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:56:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.065, "last_frame_pushed_at": "2026-07-02T09:56:18+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-02T09:56:18+09:00", "mjpeg": {"age_sec": 0.05, "exists": true, "mtime": "2026-07-02T09:56:18.496295+09:00", "mtime_epoch": 1782953778.4962952, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186225}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T09:56:18+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 874.727, "name": "celery", "pid": 237, "rss_mb": 872.406, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 134, "pushed_frames": 134, "read_frames": 135, "select_timeouts": 15, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.886, "pushed": 4.886, "read": 4.923}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.271, "count": 134, "max_ms": 2.123, "p95_ms": 1.715}, "gstreamer_push": {"avg_ms": 1.572, "count": 134, "max_ms": 6.005, "p95_ms": 3.825}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.52, "count": 134, "max_ms": 68.572, "p95_ms": 65.405}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.985, "count": 134, "max_ms": 83.206, "p95_ms": 78.056}, "yolo11_640_fire_11class": {"avg_ms": 55.143, "count": 134, "max_ms": 70.509, "p95_ms": 66.947}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.278, "count": 134, "max_ms": 78.366, "p95_ms": 75.646}}, "truncated_count": 0}, "processing_total": {"avg_ms": 74.102, "count": 134, "max_ms": 93.489, "p95_ms": 88.584}, "read_interval": {"avg_ms": 226.58, "count": 135, "max_ms": 3809.726, "p95_ms": 600.829}, "triton_wait": {"avg_ms": 63.918, "count": 134, "max_ms": 83.22, "p95_ms": 78.071}, "visualize": {"avg_ms": 3.738, "count": 134, "max_ms": 5.058, "p95_ms": 4.534}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.423, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 159}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3922.934, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T09:56:18+09:00", "last_error_summary": "[h264 @ 0x55c02a1a9b00] error while decoding MB 29 36, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:56:18+09:00", "last_error_summary": "[h264 @ 0x55c02a1a9b00] error while decoding MB 29 36, bytestream -5", "last_exit_at": "2026-07-02T09:56:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 590.704, "last_started_at": "2026-07-02T09:46:27+09:00", "pid": 1623, "process": {"exists": false, "pid": 1623}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 90.436, "sample_count": 2560, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 19091, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:56:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T09:56:18+09:00", "last_frame_read_age_sec": 0.076, "last_frame_read_at": "2026-07-02T09:56:18+09:00", "mjpeg": {"age_sec": 0.074, "exists": true, "mtime": "2026-07-02T09:56:18.496295+09:00", "mtime_epoch": 1782953778.4962952, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186225}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T09:56:18+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 874.727, "name": "celery", "pid": 237, "rss_mb": 872.406, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 135, "pushed_frames": 135, "read_frames": 135, "select_timeouts": 15, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.919, "pushed": 4.919, "read": 4.919}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.274, "count": 135, "max_ms": 2.123, "p95_ms": 1.715}, "gstreamer_push": {"avg_ms": 1.571, "count": 135, "max_ms": 6.005, "p95_ms": 3.825}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.451, "count": 135, "max_ms": 68.572, "p95_ms": 65.405}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.952, "count": 135, "max_ms": 83.206, "p95_ms": 78.056}, "yolo11_640_fire_11class": {"avg_ms": 55.076, "count": 135, "max_ms": 70.509, "p95_ms": 66.947}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.176, "count": 135, "max_ms": 78.366, "p95_ms": 75.646}}, "truncated_count": 0}, "processing_total": {"avg_ms": 74.056, "count": 135, "max_ms": 93.489, "p95_ms": 88.584}, "read_interval": {"avg_ms": 226.58, "count": 135, "max_ms": 3809.726, "p95_ms": 600.829}, "triton_wait": {"avg_ms": 63.863, "count": 135, "max_ms": 83.22, "p95_ms": 78.071}, "visualize": {"avg_ms": 3.747, "count": 135, "max_ms": 5.058, "p95_ms": 4.593}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.447, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 159}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3922.996, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:56:18+09:00", "last_error_summary": "[h264 @ 0x55c02a1a9b00] error while decoding MB 29 36, bytestream -5", "last_exit_at": "2026-07-02T09:56:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 590.712, "last_started_at": "2026-07-02T09:46:27+09:00", "pid": 1623, "process": {"exists": false, "pid": 1623}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 90.436, "sample_count": 2560, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 19091, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:56:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-02T09:56:18+09:00", "last_frame_read_age_sec": 0.084, "last_frame_read_at": "2026-07-02T09:56:18+09:00", "mjpeg": {"age_sec": 0.082, "exists": true, "mtime": "2026-07-02T09:56:18.496295+09:00", "mtime_epoch": 1782953778.4962952, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186225}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T09:56:18+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 874.727, "name": "celery", "pid": 237, "rss_mb": 872.406, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 135, "pushed_frames": 135, "read_frames": 135, "select_timeouts": 15, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.917, "pushed": 4.917, "read": 4.917}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.274, "count": 135, "max_ms": 2.123, "p95_ms": 1.715}, "gstreamer_push": {"avg_ms": 1.571, "count": 135, "max_ms": 6.005, "p95_ms": 3.825}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.451, "count": 135, "max_ms": 68.572, "p95_ms": 65.405}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.952, "count": 135, "max_ms": 83.206, "p95_ms": 78.056}, "yolo11_640_fire_11class": {"avg_ms": 55.076, "count": 135, "max_ms": 70.509, "p95_ms": 66.947}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.176, "count": 135, "max_ms": 78.366, "p95_ms": 75.646}}, "truncated_count": 0}, "processing_total": {"avg_ms": 74.056, "count": 135, "max_ms": 93.489, "p95_ms": 88.584}, "read_interval": {"avg_ms": 226.58, "count": 135, "max_ms": 3809.726, "p95_ms": 600.829}, "triton_wait": {"avg_ms": 63.863, "count": 135, "max_ms": 83.22, "p95_ms": 78.071}, "visualize": {"avg_ms": 3.747, "count": 135, "max_ms": 5.058, "p95_ms": 4.593}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 27.455, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 159}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4028.52, "limit_mb": 8796093022207.996}, "detail": {"pid": 1730, "restart_count": 4, "start_count": 5}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:56:18+09:00", "last_error_summary": "[h264 @ 0x55c02a1a9b00] error while decoding MB 29 36, bytestream -5", "last_exit_at": "2026-07-02T09:56:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T09:56:20+09:00", "pid": 1730, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.207, "name": "ffmpeg", "pid": 1730, "rss_mb": 136.207, "state": "S (sleeping)", "threads": 106}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 90.364, "sample_count": 2515, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 19091, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:56:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.32, "last_frame_pushed_at": "2026-07-02T09:56:18+09:00", "last_frame_read_age_sec": 4.39, "last_frame_read_at": "2026-07-02T09:56:18+09:00", "mjpeg": {"age_sec": 4.304, "exists": true, "mtime": "2026-07-02T09:56:18.580295+09:00", "mtime_epoch": 1782953778.580295, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186872}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T09:56:22+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 874.727, "name": "celery", "pid": 237, "rss_mb": 872.406, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 116, "select_timeouts": 12, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.902, "pushed": 3.902, "read": 3.868}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.248, "count": 117, "max_ms": 1.789, "p95_ms": 1.571}, "gstreamer_push": {"avg_ms": 1.538, "count": 117, "max_ms": 6.005, "p95_ms": 3.734}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.365, "count": 117, "max_ms": 68.572, "p95_ms": 65.451}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.319, "count": 117, "max_ms": 83.206, "p95_ms": 77.35}, "yolo11_640_fire_11class": {"avg_ms": 54.782, "count": 117, "max_ms": 70.509, "p95_ms": 67.303}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.535, "count": 117, "max_ms": 77.254, "p95_ms": 74.714}}, "truncated_count": 0}, "processing_total": {"avg_ms": 73.354, "count": 117, "max_ms": 93.342, "p95_ms": 87.514}, "read_interval": {"avg_ms": 221.207, "count": 116, "max_ms": 3809.726, "p95_ms": 600.829}, "triton_wait": {"avg_ms": 63.246, "count": 117, "max_ms": 83.22, "p95_ms": 77.362}, "visualize": {"avg_ms": 3.724, "count": 117, "max_ms": 5.058, "p95_ms": 4.593}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.986, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 162}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3919.246, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5638129486c0] error while decoding MB 79 20, bytestream -24 [h264 @ 0x5638128eebc0] error while decoding MB 16 38, bytestream -11", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:58:03+09:00", "last_error_summary": "[h264 @ 0x5638129486c0] error while decoding MB 79 20, bytestream -24 [h264 @ 0x5638128eebc0] error while decoding MB 16 38, bytestream -11", "last_exit_at": "2026-07-02T09:56:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 102.234, "last_started_at": "2026-07-02T09:56:20+09:00", "pid": 1730, "process": {"exists": false, "pid": 1730}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 89.939, "sample_count": 1873, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 19455, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:58:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-02T09:58:03+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-02T09:58:03+09:00", "mjpeg": {"age_sec": 0.049, "exists": true, "mtime": "2026-07-02T09:58:03.068371+09:00", "mtime_epoch": 1782953883.0683706, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185018}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T09:58:03+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 874.727, "name": "celery", "pid": 237, "rss_mb": 867.25, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 124, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.142, "pushed": 4.142, "read": 4.176}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.236, "count": 123, "max_ms": 1.759, "p95_ms": 1.268}, "gstreamer_push": {"avg_ms": 1.52, "count": 123, "max_ms": 6.142, "p95_ms": 3.321}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.502, "count": 123, "max_ms": 64.027, "p95_ms": 51.85}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.759, "count": 123, "max_ms": 78.457, "p95_ms": 58.502}, "yolo11_640_fire_11class": {"avg_ms": 46.838, "count": 123, "max_ms": 65.772, "p95_ms": 51.24}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.063, "count": 123, "max_ms": 77.196, "p95_ms": 54.886}}, "truncated_count": 0}, "processing_total": {"avg_ms": 65.191, "count": 123, "max_ms": 88.272, "p95_ms": 68.842}, "read_interval": {"avg_ms": 242.903, "count": 124, "max_ms": 10100.128, "p95_ms": 405.764}, "triton_wait": {"avg_ms": 55.257, "count": 123, "max_ms": 78.469, "p95_ms": 59.02}, "visualize": {"avg_ms": 3.692, "count": 123, "max_ms": 4.197, "p95_ms": 3.913}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.696, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 162}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3921.66, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T09:58:03+09:00", "last_error_summary": "[h264 @ 0x5638129486c0] error while decoding MB 79 20, bytestream -24 [h264 @ 0x5638128eebc0] error while decoding MB 16 38, bytestream -11", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:58:03+09:00", "last_error_summary": "[h264 @ 0x5638129486c0] error while decoding MB 79 20, bytestream -24 [h264 @ 0x5638128eebc0] error while decoding MB 16 38, bytestream -11", "last_exit_at": "2026-07-02T09:58:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 102.26, "last_started_at": "2026-07-02T09:56:20+09:00", "pid": 1730, "process": {"exists": false, "pid": 1730}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 89.939, "sample_count": 1873, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 19456, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:58:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T09:58:03+09:00", "last_frame_read_age_sec": 0.077, "last_frame_read_at": "2026-07-02T09:58:03+09:00", "mjpeg": {"age_sec": 0.075, "exists": true, "mtime": "2026-07-02T09:58:03.068371+09:00", "mtime_epoch": 1782953883.0683706, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185018}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T09:58:03+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 874.727, "name": "celery", "pid": 237, "rss_mb": 869.828, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 124, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.172, "pushed": 4.172, "read": 4.172}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.238, "count": 124, "max_ms": 1.759, "p95_ms": 1.274}, "gstreamer_push": {"avg_ms": 1.539, "count": 124, "max_ms": 6.142, "p95_ms": 3.596}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.492, "count": 124, "max_ms": 64.027, "p95_ms": 51.85}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.764, "count": 124, "max_ms": 78.457, "p95_ms": 58.502}, "yolo11_640_fire_11class": {"avg_ms": 46.829, "count": 124, "max_ms": 65.772, "p95_ms": 51.24}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.061, "count": 124, "max_ms": 77.196, "p95_ms": 54.886}}, "truncated_count": 0}, "processing_total": {"avg_ms": 65.206, "count": 124, "max_ms": 88.272, "p95_ms": 68.842}, "read_interval": {"avg_ms": 242.903, "count": 124, "max_ms": 10100.128, "p95_ms": 405.764}, "triton_wait": {"avg_ms": 55.258, "count": 124, "max_ms": 78.469, "p95_ms": 59.02}, "visualize": {"avg_ms": 3.706, "count": 124, "max_ms": 5.431, "p95_ms": 3.963}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.721, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 162}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3921.75, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:58:03+09:00", "last_error_summary": "[h264 @ 0x5638129486c0] error while decoding MB 79 20, bytestream -24 [h264 @ 0x5638128eebc0] error while decoding MB 16 38, bytestream -11", "last_exit_at": "2026-07-02T09:58:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 102.267, "last_started_at": "2026-07-02T09:56:20+09:00", "pid": 1730, "process": {"exists": false, "pid": 1730}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 89.939, "sample_count": 1873, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 19456, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:58:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-02T09:58:03+09:00", "last_frame_read_age_sec": 0.085, "last_frame_read_at": "2026-07-02T09:58:03+09:00", "mjpeg": {"age_sec": 0.083, "exists": true, "mtime": "2026-07-02T09:58:03.068371+09:00", "mtime_epoch": 1782953883.0683706, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185018}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T09:58:03+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 874.727, "name": "celery", "pid": 237, "rss_mb": 869.828, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 124, "pushed_frames": 124, "read_frames": 124, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.171, "pushed": 4.171, "read": 4.171}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.238, "count": 124, "max_ms": 1.759, "p95_ms": 1.274}, "gstreamer_push": {"avg_ms": 1.539, "count": 124, "max_ms": 6.142, "p95_ms": 3.596}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.492, "count": 124, "max_ms": 64.027, "p95_ms": 51.85}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.764, "count": 124, "max_ms": 78.457, "p95_ms": 58.502}, "yolo11_640_fire_11class": {"avg_ms": 46.829, "count": 124, "max_ms": 65.772, "p95_ms": 51.24}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.061, "count": 124, "max_ms": 77.196, "p95_ms": 54.886}}, "truncated_count": 0}, "processing_total": {"avg_ms": 65.206, "count": 124, "max_ms": 88.272, "p95_ms": 68.842}, "read_interval": {"avg_ms": 242.903, "count": 124, "max_ms": 10100.128, "p95_ms": 405.764}, "triton_wait": {"avg_ms": 55.258, "count": 124, "max_ms": 78.469, "p95_ms": 59.02}, "visualize": {"avg_ms": 3.706, "count": 124, "max_ms": 5.431, "p95_ms": 3.963}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.729, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 162}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4021.051, "limit_mb": 8796093022207.996}, "detail": {"pid": 1837, "restart_count": 5, "start_count": 6}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:58:03+09:00", "last_error_summary": "[h264 @ 0x5638129486c0] error while decoding MB 79 20, bytestream -24 [h264 @ 0x5638128eebc0] error while decoding MB 16 38, bytestream -11", "last_exit_at": "2026-07-02T09:58:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T09:58:05+09:00", "pid": 1837, "process": {"exists": true, "fd_count": 5, "hwm_mb": 131.281, "name": "ffmpeg", "pid": 1837, "rss_mb": 131.281, "state": "S (sleeping)", "threads": 106}, "restart_count": 5, "restart_count_per_hour": 5, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 89.962, "sample_count": 1826, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 19456, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T09:58:03+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.86, "last_frame_pushed_at": "2026-07-02T09:58:03+09:00", "last_frame_read_age_sec": 4.932, "last_frame_read_at": "2026-07-02T09:58:03+09:00", "mjpeg": {"age_sec": 4.846, "exists": true, "mtime": "2026-07-02T09:58:03.152371+09:00", "mtime_epoch": 1782953883.1523707, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185299}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T09:58:08+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 874.727, "name": "celery", "pid": 237, "rss_mb": 869.828, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 94, "pushed_frames": 94, "read_frames": 94, "select_timeouts": 15, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.203, "pushed": 3.203, "read": 3.203}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.234, "count": 94, "max_ms": 1.593, "p95_ms": 1.282}, "gstreamer_push": {"avg_ms": 1.518, "count": 94, "max_ms": 5.874, "p95_ms": 3.596}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.608, "count": 94, "max_ms": 64.027, "p95_ms": 61.097}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.026, "count": 94, "max_ms": 78.457, "p95_ms": 59.995}, "yolo11_640_fire_11class": {"avg_ms": 46.929, "count": 94, "max_ms": 65.772, "p95_ms": 63.222}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.354, "count": 94, "max_ms": 77.196, "p95_ms": 70.504}}, "truncated_count": 0}, "processing_total": {"avg_ms": 65.591, "count": 94, "max_ms": 88.272, "p95_ms": 80.831}, "read_interval": {"avg_ms": 291.137, "count": 94, "max_ms": 10100.128, "p95_ms": 565.845}, "triton_wait": {"avg_ms": 55.674, "count": 94, "max_ms": 78.469, "p95_ms": 70.966}, "visualize": {"avg_ms": 3.711, "count": 94, "max_ms": 5.431, "p95_ms": 3.989}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.352, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 168}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3925.199, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T09:58:03+09:00", "last_error_summary": "[h264 @ 0x5638129486c0] error while decoding MB 79 20, bytestream -24 [h264 @ 0x5638128eebc0] error while decoding MB 16 38, bytestream -11", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:58:03+09:00", "last_error_summary": "[h264 @ 0x5638129486c0] error while decoding MB 79 20, bytestream -24 [h264 @ 0x5638128eebc0] error while decoding MB 16 38, bytestream -11", "last_exit_at": "2026-07-02T10:00:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 126.891, "last_started_at": "2026-07-02T09:58:05+09:00", "pid": 1837, "process": {"exists": false, "pid": 1837}, "restart_count": 5, "restart_count_per_hour": 5, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 88.542, "sample_count": 1435, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 20205, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T10:00:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T10:00:12+09:00", "last_frame_read_age_sec": 0.097, "last_frame_read_at": "2026-07-02T10:00:12+09:00", "mjpeg": {"age_sec": 0.095, "exists": true, "mtime": "2026-07-02T10:00:12.792698+09:00", "mtime_epoch": 1782954012.7926984, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189477}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T10:00:12+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 874.727, "name": "celery", "pid": 237, "rss_mb": 872.816, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 206, "pushed_frames": 207, "read_frames": 206, "select_timeouts": 9, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.867, "pushed": 6.9, "read": 6.867}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.298, "count": 206, "max_ms": 3.425, "p95_ms": 1.352}, "gstreamer_push": {"avg_ms": 1.635, "count": 207, "max_ms": 6.389, "p95_ms": 3.841}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.393, "count": 206, "max_ms": 67.593, "p95_ms": 64.343}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.223, "count": 206, "max_ms": 83.279, "p95_ms": 79.035}, "yolo11_640_fire_11class": {"avg_ms": 55.337, "count": 206, "max_ms": 70.885, "p95_ms": 67.547}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.804, "count": 206, "max_ms": 81.704, "p95_ms": 78.378}}, "truncated_count": 0}, "processing_total": {"avg_ms": 75.361, "count": 206, "max_ms": 103.783, "p95_ms": 90.382}, "read_interval": {"avg_ms": 145.473, "count": 206, "max_ms": 3821.15, "p95_ms": 115.334}, "triton_wait": {"avg_ms": 64.88, "count": 206, "max_ms": 83.292, "p95_ms": 79.398}, "visualize": {"avg_ms": 3.706, "count": 206, "max_ms": 5.144, "p95_ms": 3.855}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 168}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 3925.176, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:58:03+09:00", "last_error_summary": "[h264 @ 0x5638129486c0] error while decoding MB 79 20, bytestream -24 [h264 @ 0x5638128eebc0] error while decoding MB 16 38, bytestream -11", "last_exit_at": "2026-07-02T10:00:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 126.899, "last_started_at": "2026-07-02T09:58:05+09:00", "pid": 1837, "process": {"exists": false, "pid": 1837}, "restart_count": 5, "restart_count_per_hour": 5, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 88.542, "sample_count": 1435, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 20205, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T10:00:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-02T10:00:12+09:00", "last_frame_read_age_sec": 0.105, "last_frame_read_at": "2026-07-02T10:00:12+09:00", "mjpeg": {"age_sec": 0.103, "exists": true, "mtime": "2026-07-02T10:00:12.792698+09:00", "mtime_epoch": 1782954012.7926984, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189477}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T10:00:12+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 874.727, "name": "celery", "pid": 237, "rss_mb": 872.816, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 206, "pushed_frames": 206, "read_frames": 206, "select_timeouts": 9, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.868, "pushed": 6.868, "read": 6.868}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.298, "count": 206, "max_ms": 3.425, "p95_ms": 1.352}, "gstreamer_push": {"avg_ms": 1.636, "count": 206, "max_ms": 6.389, "p95_ms": 3.841}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.393, "count": 206, "max_ms": 67.593, "p95_ms": 64.343}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.223, "count": 206, "max_ms": 83.279, "p95_ms": 79.035}, "yolo11_640_fire_11class": {"avg_ms": 55.337, "count": 206, "max_ms": 70.885, "p95_ms": 67.547}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.804, "count": 206, "max_ms": 81.704, "p95_ms": 78.378}}, "truncated_count": 0}, "processing_total": {"avg_ms": 75.361, "count": 206, "max_ms": 103.783, "p95_ms": 90.382}, "read_interval": {"avg_ms": 145.473, "count": 206, "max_ms": 3821.15, "p95_ms": 115.334}, "triton_wait": {"avg_ms": 64.88, "count": 206, "max_ms": 83.292, "p95_ms": 79.398}, "visualize": {"avg_ms": 3.706, "count": 206, "max_ms": 5.144, "p95_ms": 3.855}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 168}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4030.477, "limit_mb": 8796093022207.996}, "detail": {"pid": 1944, "restart_count": 6, "start_count": 7}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T09:58:03+09:00", "last_error_summary": "[h264 @ 0x5638129486c0] error while decoding MB 79 20, bytestream -24 [h264 @ 0x5638128eebc0] error while decoding MB 16 38, bytestream -11", "last_exit_at": "2026-07-02T10:00:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T10:00:15+09:00", "pid": 1944, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.566, "name": "ffmpeg", "pid": 1944, "rss_mb": 136.566, "state": "S (sleeping)", "threads": 106}, "restart_count": 6, "restart_count_per_hour": 6, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 88.542, "sample_count": 1434, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 20205, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T10:00:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.779, "last_frame_pushed_at": "2026-07-02T10:00:12+09:00", "last_frame_read_age_sec": 4.87, "last_frame_read_at": "2026-07-02T10:00:12+09:00", "mjpeg": {"age_sec": 4.764, "exists": true, "mtime": "2026-07-02T10:00:12.896699+09:00", "mtime_epoch": 1782954012.8966987, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187729}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T10:00:17+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 874.727, "name": "celery", "pid": 237, "rss_mb": 872.816, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 179, "pushed_frames": 179, "read_frames": 178, "select_timeouts": 7, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.969, "pushed": 5.969, "read": 5.936}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.305, "count": 179, "max_ms": 3.425, "p95_ms": 1.37}, "gstreamer_push": {"avg_ms": 1.671, "count": 179, "max_ms": 6.389, "p95_ms": 3.869}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.609, "count": 179, "max_ms": 67.593, "p95_ms": 64.5}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.338, "count": 179, "max_ms": 83.279, "p95_ms": 79.386}, "yolo11_640_fire_11class": {"avg_ms": 55.586, "count": 179, "max_ms": 70.885, "p95_ms": 67.797}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.239, "count": 179, "max_ms": 81.704, "p95_ms": 78.434}}, "truncated_count": 0}, "processing_total": {"avg_ms": 75.756, "count": 179, "max_ms": 103.783, "p95_ms": 90.609}, "read_interval": {"avg_ms": 141.465, "count": 178, "max_ms": 3821.15, "p95_ms": 162.846}, "triton_wait": {"avg_ms": 65.197, "count": 179, "max_ms": 83.292, "p95_ms": 80.141}, "visualize": {"avg_ms": 3.705, "count": 179, "max_ms": 5.144, "p95_ms": 3.854}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.986, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/25/tracking_analysis_20260702_100226_6f4a2941.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 0}, "started": true}, "buffers": {"frame_buffer_len": 0, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "25", "cgroup_memory": {"current_mb": 4280.375, "limit_mb": 8796093022207.996}, "detail": {"pid": 2054, "restart_count": 0, "start_count": 1}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 0, "h264_decode_error_count": 0, "h264_decode_error_count_per_hour": 0, "input_options": [], "last_error_at": "", "last_error_summary": "", "last_exit_at": "", "last_returncode": null, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T10:02:26+09:00", "pid": 2054, "process": {"exists": true, "fd_count": 5, "hwm_mb": 137.602, "name": "ffmpeg", "pid": 2054, "rss_mb": 137.602, "state": "S (sleeping)", "threads": 106}, "restart_count": 0, "restart_count_per_hour": 0, "rtsp_timeout_count": 0, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 1}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": null, "sample_count": 0, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 297.796}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 0, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["mjpeg_age_sec>30s"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "6f4a2941", "last_frame_pushed_age_sec": null, "last_frame_pushed_at": "", "last_frame_read_age_sec": null, "last_frame_read_at": "", "mjpeg": {"age_sec": 2176.473, "exists": true, "mtime": "2026-07-02T09:26:12.107144+09:00", "mtime_epoch": 1782951972.1071444, "path": "/data/playlist/streaming/25/mjpeg.jpg", "size_bytes": 250722}, "monitor_url": "http://192.168.50.244:1223/mjpeg/25", "observed_at": "2026-07-02T10:02:28+09:00", "process": {"exists": true, "fd_count": 117, "hwm_mb": 699.418, "name": "celery", "pid": 239, "rss_mb": 673.613, "state": "R (running)", "threads": 118}, "reason_codes": ["mjpeg_output_stale_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "file:///data/box_steam/260423_seochosite_dataset/input/video/20260507_155115_260507_샘플다시저장_원본영상_sample_live_720p_10fps_aligned/260511_6분30초부터_8분15초까지_clip/clip_0390-0495s/2026-04-23 12_59_58 이동형 #1__live_camfix_1280x720_10fps_crf23__clip_0390-0495s.mp4", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 0, "pushed_frames": 0, "read_frames": 0, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.0, "pushed": 0.0, "read": 0.0}, "hints": [], "latency_ms": {"analyze": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "gstreamer_push": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "model_completion": {"model_count": 0, "models": {}, "truncated_count": 0}, "processing_total": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "read_interval": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "triton_wait": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}, "visualize": {"avg_ms": null, "count": 0, "max_ms": null, "p95_ms": null}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-02T10:02:26+09:00", "state": "running", "task_id": "665d2126-01ff-4c54-a771-0197b6a64487", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 205}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4239.086, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 8, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55ee7aa42ec0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ee7aa42ec0] error while decoding MB 0 39, bytestream 4835 [h264 @ 0x55ee7b491f40] cabac decode of qscale diff failed at 71 44 [h264 @ 0x55ee7b491f40] error while decoding MB 71 44, bytestream 2313 [h264 @ 0x55ee7aa9a640] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ee7aa9a640] error while decoding MB 0 41, bytestream 78355 [h264 @ 0x55ee7b6feac0] left block unavailable for requested int", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 13, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T10:08:18+09:00", "last_error_summary": "[h264 @ 0x55ee7aa42ec0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ee7aa42ec0] error while decoding MB 0 39, bytestream 4835 [h264 @ 0x55ee7b491f40] cabac decode of qscale diff failed at 71 44 [h264 @ 0x55ee7b491f40] error while decoding MB 71 44, bytestream 2313 [h264 @ 0x55ee7aa9a640] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ee7aa9a640] error while decoding MB 0 41, bytestream 78355 [h264 @ 0x55ee7b6feac0] left block unavailable for requested int", "last_exit_at": "2026-07-02T10:00:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 483.017, "last_started_at": "2026-07-02T10:00:15+09:00", "pid": 1944, "process": {"exists": false, "pid": 1944}, "restart_count": 6, "restart_count_per_hour": 6, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 132.983, "sample_count": 2691, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 24704, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T10:08:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.073, "last_frame_pushed_at": "2026-07-02T10:08:18+09:00", "last_frame_read_age_sec": 0.06, "last_frame_read_at": "2026-07-02T10:08:18+09:00", "mjpeg": {"age_sec": 0.054, "exists": true, "mtime": "2026-07-02T10:08:18.623409+09:00", "mtime_epoch": 1782954498.6234086, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195183}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T10:08:18+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 880.539, "name": "celery", "pid": 237, "rss_mb": 872.742, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 108, "pushed_frames": 108, "read_frames": 109, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.601, "pushed": 3.601, "read": 3.634}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.608, "count": 108, "max_ms": 2.225, "p95_ms": 2.158}, "gstreamer_push": {"avg_ms": 1.608, "count": 108, "max_ms": 5.811, "p95_ms": 3.355}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.019, "count": 108, "max_ms": 82.198, "p95_ms": 75.914}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.467, "count": 108, "max_ms": 107.699, "p95_ms": 95.469}, "yolo11_640_fire_11class": {"avg_ms": 60.609, "count": 108, "max_ms": 84.342, "p95_ms": 78.358}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.193, "count": 108, "max_ms": 92.136, "p95_ms": 85.663}}, "truncated_count": 0}, "processing_total": {"avg_ms": 82.286, "count": 108, "max_ms": 118.513, "p95_ms": 106.149}, "read_interval": {"avg_ms": 275.565, "count": 109, "max_ms": 19224.278, "p95_ms": 132.23}, "triton_wait": {"avg_ms": 71.302, "count": 108, "max_ms": 107.714, "p95_ms": 95.48}, "visualize": {"avg_ms": 3.934, "count": 108, "max_ms": 6.9, "p95_ms": 6.674}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 205}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4238.645, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T10:08:18+09:00", "last_error_summary": "[h264 @ 0x55ee7aa42ec0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ee7aa42ec0] error while decoding MB 0 39, bytestream 4835 [h264 @ 0x55ee7b491f40] cabac decode of qscale diff failed at 71 44 [h264 @ 0x55ee7b491f40] error while decoding MB 71 44, bytestream 2313 [h264 @ 0x55ee7aa9a640] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ee7aa9a640] error while decoding MB 0 41, bytestream 78355 [h264 @ 0x55ee7b6feac0] left block unavailable for requested int", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 13, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T10:08:18+09:00", "last_error_summary": "[h264 @ 0x55ee7aa42ec0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ee7aa42ec0] error while decoding MB 0 39, bytestream 4835 [h264 @ 0x55ee7b491f40] cabac decode of qscale diff failed at 71 44 [h264 @ 0x55ee7b491f40] error while decoding MB 71 44, bytestream 2313 [h264 @ 0x55ee7aa9a640] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ee7aa9a640] error while decoding MB 0 41, bytestream 78355 [h264 @ 0x55ee7b6feac0] left block unavailable for requested int", "last_exit_at": "2026-07-02T10:08:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 483.038, "last_started_at": "2026-07-02T10:00:15+09:00", "pid": 1944, "process": {"exists": false, "pid": 1944}, "restart_count": 6, "restart_count_per_hour": 6, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 132.964, "sample_count": 2692, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 24705, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T10:08:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T10:08:18+09:00", "last_frame_read_age_sec": 0.08, "last_frame_read_at": "2026-07-02T10:08:18+09:00", "mjpeg": {"age_sec": 0.074, "exists": true, "mtime": "2026-07-02T10:08:18.623409+09:00", "mtime_epoch": 1782954498.6234086, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195183}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T10:08:18+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 880.539, "name": "celery", "pid": 237, "rss_mb": 872.746, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 109, "pushed_frames": 109, "read_frames": 108, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.642, "pushed": 3.642, "read": 3.609}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.608, "count": 109, "max_ms": 2.225, "p95_ms": 2.158}, "gstreamer_push": {"avg_ms": 1.605, "count": 109, "max_ms": 5.811, "p95_ms": 3.355}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.951, "count": 109, "max_ms": 82.198, "p95_ms": 75.914}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.364, "count": 109, "max_ms": 107.699, "p95_ms": 95.469}, "yolo11_640_fire_11class": {"avg_ms": 60.523, "count": 109, "max_ms": 84.342, "p95_ms": 78.358}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.083, "count": 109, "max_ms": 92.136, "p95_ms": 85.663}}, "truncated_count": 0}, "processing_total": {"avg_ms": 82.187, "count": 109, "max_ms": 118.513, "p95_ms": 106.149}, "read_interval": {"avg_ms": 277.15, "count": 108, "max_ms": 19224.278, "p95_ms": 132.23}, "triton_wait": {"avg_ms": 71.164, "count": 109, "max_ms": 107.714, "p95_ms": 95.48}, "visualize": {"avg_ms": 3.964, "count": 109, "max_ms": 7.215, "p95_ms": 6.677}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.925, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 205}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4238.461, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 13, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T10:08:18+09:00", "last_error_summary": "[h264 @ 0x55ee7aa42ec0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ee7aa42ec0] error while decoding MB 0 39, bytestream 4835 [h264 @ 0x55ee7b491f40] cabac decode of qscale diff failed at 71 44 [h264 @ 0x55ee7b491f40] error while decoding MB 71 44, bytestream 2313 [h264 @ 0x55ee7aa9a640] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ee7aa9a640] error while decoding MB 0 41, bytestream 78355 [h264 @ 0x55ee7b6feac0] left block unavailable for requested int", "last_exit_at": "2026-07-02T10:08:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 483.047, "last_started_at": "2026-07-02T10:00:15+09:00", "pid": 1944, "process": {"exists": false, "pid": 1944}, "restart_count": 6, "restart_count_per_hour": 6, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 132.964, "sample_count": 2692, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 24705, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T10:08:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-02T10:08:18+09:00", "last_frame_read_age_sec": 0.089, "last_frame_read_at": "2026-07-02T10:08:18+09:00", "mjpeg": {"age_sec": 0.083, "exists": true, "mtime": "2026-07-02T10:08:18.623409+09:00", "mtime_epoch": 1782954498.6234086, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195183}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T10:08:18+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 880.539, "name": "celery", "pid": 237, "rss_mb": 872.746, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 109, "pushed_frames": 109, "read_frames": 108, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.641, "pushed": 3.641, "read": 3.608}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.608, "count": 109, "max_ms": 2.225, "p95_ms": 2.158}, "gstreamer_push": {"avg_ms": 1.605, "count": 109, "max_ms": 5.811, "p95_ms": 3.355}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.951, "count": 109, "max_ms": 82.198, "p95_ms": 75.914}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.364, "count": 109, "max_ms": 107.699, "p95_ms": 95.469}, "yolo11_640_fire_11class": {"avg_ms": 60.523, "count": 109, "max_ms": 84.342, "p95_ms": 78.358}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.083, "count": 109, "max_ms": 92.136, "p95_ms": 85.663}}, "truncated_count": 0}, "processing_total": {"avg_ms": 82.187, "count": 109, "max_ms": 118.513, "p95_ms": 106.149}, "read_interval": {"avg_ms": 277.15, "count": 108, "max_ms": 19224.278, "p95_ms": 132.23}, "triton_wait": {"avg_ms": 71.164, "count": 109, "max_ms": 107.714, "p95_ms": 95.48}, "visualize": {"avg_ms": 3.964, "count": 109, "max_ms": 7.215, "p95_ms": 6.677}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.934, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 205}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4343.242, "limit_mb": 8796093022207.996}, "detail": {"pid": 2207, "restart_count": 7, "start_count": 8}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 13, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T10:08:18+09:00", "last_error_summary": "[h264 @ 0x55ee7aa42ec0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ee7aa42ec0] error while decoding MB 0 39, bytestream 4835 [h264 @ 0x55ee7b491f40] cabac decode of qscale diff failed at 71 44 [h264 @ 0x55ee7b491f40] error while decoding MB 71 44, bytestream 2313 [h264 @ 0x55ee7aa9a640] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ee7aa9a640] error while decoding MB 0 41, bytestream 78355 [h264 @ 0x55ee7b6feac0] left block unavailable for requested int", "last_exit_at": "2026-07-02T10:08:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T10:08:21+09:00", "pid": 2207, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.258, "name": "ffmpeg", "pid": 2207, "rss_mb": 136.258, "state": "S (sleeping)", "threads": 106}, "restart_count": 7, "restart_count_per_hour": 7, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 133.508, "sample_count": 2637, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 24705, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T10:08:18+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.842, "last_frame_pushed_at": "2026-07-02T10:08:18+09:00", "last_frame_read_age_sec": 4.916, "last_frame_read_at": "2026-07-02T10:08:18+09:00", "mjpeg": {"age_sec": 4.822, "exists": true, "mtime": "2026-07-02T10:08:18.711409+09:00", "mtime_epoch": 1782954498.711409, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186953}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T10:08:23+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 880.539, "name": "celery", "pid": 237, "rss_mb": 872.746, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 61, "pushed_frames": 61, "read_frames": 61, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.034, "pushed": 2.034, "read": 2.034}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 1.65, "count": 61, "max_ms": 2.225, "p95_ms": 2.179}, "gstreamer_push": {"avg_ms": 1.72, "count": 61, "max_ms": 5.811, "p95_ms": 3.899}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 59.301, "count": 61, "max_ms": 82.198, "p95_ms": 72.498}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.997, "count": 61, "max_ms": 107.449, "p95_ms": 90.78}, "yolo11_640_fire_11class": {"avg_ms": 59.928, "count": 61, "max_ms": 84.342, "p95_ms": 75.38}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.587, "count": 61, "max_ms": 91.103, "p95_ms": 82.907}}, "truncated_count": 0}, "processing_total": {"avg_ms": 81.901, "count": 61, "max_ms": 118.114, "p95_ms": 106.116}, "read_interval": {"avg_ms": 413.037, "count": 61, "max_ms": 19224.278, "p95_ms": 127.837}, "triton_wait": {"avg_ms": 70.517, "count": 61, "max_ms": 107.463, "p95_ms": 91.114}, "visualize": {"avg_ms": 4.138, "count": 61, "max_ms": 7.215, "p95_ms": 6.7}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.986, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 335}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4247.559, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56185ee4b000] error while decoding MB 42 35, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T10:32:43+09:00", "last_error_summary": "[h264 @ 0x56185ee4b000] error while decoding MB 42 35, bytestream -5", "last_exit_at": "2026-07-02T09:42:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 3004.796, "last_started_at": "2026-07-02T09:42:38+09:00", "pid": 1257, "process": {"exists": false, "pid": 1257}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 163.289, "sample_count": 2120, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 40216, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T10:32:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.074, "last_frame_pushed_at": "2026-07-02T10:32:43+09:00", "last_frame_read_age_sec": 0.058, "last_frame_read_at": "2026-07-02T10:32:43+09:00", "mjpeg": {"age_sec": 0.057, "exists": true, "mtime": "2026-07-02T10:32:43.672284+09:00", "mtime_epoch": 1782955963.6722844, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 253841}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T10:32:43+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1147.559, "name": "celery", "pid": 267, "rss_mb": 920.801, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 189, "pushed_frames": 189, "read_frames": 190, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.301, "pushed": 6.301, "read": 6.334}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.497, "count": 189, "max_ms": 7.845, "p95_ms": 6.626}, "gstreamer_push": {"avg_ms": 1.017, "count": 189, "max_ms": 1.395, "p95_ms": 1.173}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.421, "count": 189, "max_ms": 76.159, "p95_ms": 67.4}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.506, "count": 189, "max_ms": 106.039, "p95_ms": 86.894}, "yolo11_640_fire_11class": {"avg_ms": 57.291, "count": 189, "max_ms": 78.915, "p95_ms": 70.687}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.243, "count": 189, "max_ms": 92.226, "p95_ms": 83.474}}, "truncated_count": 0}, "processing_total": {"avg_ms": 146.831, "count": 189, "max_ms": 183.909, "p95_ms": 171.153}, "read_interval": {"avg_ms": 158.319, "count": 190, "max_ms": 195.178, "p95_ms": 182.761}, "triton_wait": {"avg_ms": 66.947, "count": 189, "max_ms": 106.051, "p95_ms": 86.952}, "visualize": {"avg_ms": 6.525, "count": 189, "max_ms": 10.939, "p95_ms": 7.248}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 335}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4246.91, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T10:32:43+09:00", "last_error_summary": "[h264 @ 0x56185ee4b000] error while decoding MB 42 35, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T10:32:43+09:00", "last_error_summary": "[h264 @ 0x56185ee4b000] error while decoding MB 42 35, bytestream -5", "last_exit_at": "2026-07-02T10:32:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 3004.859, "last_started_at": "2026-07-02T09:42:38+09:00", "pid": 1257, "process": {"exists": false, "pid": 1257}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 163.289, "sample_count": 2120, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 40217, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T10:32:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T10:32:43+09:00", "last_frame_read_age_sec": 0.121, "last_frame_read_at": "2026-07-02T10:32:43+09:00", "mjpeg": {"age_sec": 0.12, "exists": true, "mtime": "2026-07-02T10:32:43.672284+09:00", "mtime_epoch": 1782955963.6722844, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 253841}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T10:32:43+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1147.559, "name": "celery", "pid": 267, "rss_mb": 920.805, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 190, "pushed_frames": 190, "read_frames": 189, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.356, "pushed": 6.356, "read": 6.322}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.492, "count": 190, "max_ms": 7.845, "p95_ms": 6.626}, "gstreamer_push": {"avg_ms": 1.018, "count": 190, "max_ms": 1.395, "p95_ms": 1.173}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.373, "count": 190, "max_ms": 76.159, "p95_ms": 67.4}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.374, "count": 190, "max_ms": 106.039, "p95_ms": 86.894}, "yolo11_640_fire_11class": {"avg_ms": 57.253, "count": 190, "max_ms": 78.915, "p95_ms": 70.687}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.176, "count": 190, "max_ms": 92.226, "p95_ms": 83.474}}, "truncated_count": 0}, "processing_total": {"avg_ms": 146.659, "count": 190, "max_ms": 183.909, "p95_ms": 171.153}, "read_interval": {"avg_ms": 158.391, "count": 189, "max_ms": 195.178, "p95_ms": 182.761}, "triton_wait": {"avg_ms": 66.859, "count": 190, "max_ms": 106.051, "p95_ms": 86.952}, "visualize": {"avg_ms": 6.537, "count": 190, "max_ms": 10.939, "p95_ms": 7.276}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.895, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 335}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4246.902, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T10:32:43+09:00", "last_error_summary": "[h264 @ 0x56185ee4b000] error while decoding MB 42 35, bytestream -5", "last_exit_at": "2026-07-02T10:32:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 3004.866, "last_started_at": "2026-07-02T09:42:38+09:00", "pid": 1257, "process": {"exists": false, "pid": 1257}, "restart_count": 2, "restart_count_per_hour": 2, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 3}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 163.289, "sample_count": 2120, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 40217, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T10:32:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-02T10:32:43+09:00", "last_frame_read_age_sec": 0.127, "last_frame_read_at": "2026-07-02T10:32:43+09:00", "mjpeg": {"age_sec": 0.126, "exists": true, "mtime": "2026-07-02T10:32:43.672284+09:00", "mtime_epoch": 1782955963.6722844, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 253841}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T10:32:43+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1147.559, "name": "celery", "pid": 267, "rss_mb": 920.805, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 190, "pushed_frames": 190, "read_frames": 189, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.354, "pushed": 6.354, "read": 6.321}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.492, "count": 190, "max_ms": 7.845, "p95_ms": 6.626}, "gstreamer_push": {"avg_ms": 1.018, "count": 190, "max_ms": 1.395, "p95_ms": 1.173}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.373, "count": 190, "max_ms": 76.159, "p95_ms": 67.4}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.374, "count": 190, "max_ms": 106.039, "p95_ms": 86.894}, "yolo11_640_fire_11class": {"avg_ms": 57.253, "count": 190, "max_ms": 78.915, "p95_ms": 70.687}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.176, "count": 190, "max_ms": 92.226, "p95_ms": 83.474}}, "truncated_count": 0}, "processing_total": {"avg_ms": 146.659, "count": 190, "max_ms": 183.909, "p95_ms": 171.153}, "read_interval": {"avg_ms": 158.391, "count": 189, "max_ms": 195.178, "p95_ms": 182.761}, "triton_wait": {"avg_ms": 66.859, "count": 190, "max_ms": 106.051, "p95_ms": 86.952}, "visualize": {"avg_ms": 6.537, "count": 190, "max_ms": 10.939, "p95_ms": 7.276}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.901, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 335}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4320.5, "limit_mb": 8796093022207.996}, "detail": {"pid": 2374, "restart_count": 3, "start_count": 4}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 3, "h264_decode_error_count": 3, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T10:32:43+09:00", "last_error_summary": "[h264 @ 0x56185ee4b000] error while decoding MB 42 35, bytestream -5", "last_exit_at": "2026-07-02T10:32:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T10:32:46+09:00", "pid": 2374, "process": {"exists": true, "fd_count": 5, "hwm_mb": 105.488, "name": "ffmpeg", "pid": 2374, "rss_mb": 105.488, "state": "S (sleeping)", "threads": 19}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 163.137, "sample_count": 2092, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 40217, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T10:32:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 4.245, "last_frame_pushed_at": "2026-07-02T10:32:43+09:00", "last_frame_read_age_sec": 4.361, "last_frame_read_at": "2026-07-02T10:32:43+09:00", "mjpeg": {"age_sec": 4.228, "exists": true, "mtime": "2026-07-02T10:32:43.804286+09:00", "mtime_epoch": 1782955963.8042855, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 249216}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T10:32:48+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1147.559, "name": "celery", "pid": 267, "rss_mb": 920.805, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 164, "pushed_frames": 164, "read_frames": 163, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.471, "pushed": 5.471, "read": 5.438}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.488, "count": 163, "max_ms": 7.779, "p95_ms": 6.626}, "gstreamer_push": {"avg_ms": 1.016, "count": 164, "max_ms": 1.395, "p95_ms": 1.162}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.244, "count": 163, "max_ms": 76.159, "p95_ms": 67.118}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.489, "count": 163, "max_ms": 106.039, "p95_ms": 86.894}, "yolo11_640_fire_11class": {"avg_ms": 57.099, "count": 163, "max_ms": 78.915, "p95_ms": 69.67}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.03, "count": 163, "max_ms": 92.226, "p95_ms": 83.474}}, "truncated_count": 0}, "processing_total": {"avg_ms": 146.231, "count": 164, "max_ms": 183.909, "p95_ms": 171.153}, "read_interval": {"avg_ms": 157.957, "count": 163, "max_ms": 195.178, "p95_ms": 182.761}, "triton_wait": {"avg_ms": 66.498, "count": 163, "max_ms": 106.051, "p95_ms": 86.952}, "visualize": {"avg_ms": 6.531, "count": 163, "max_ms": 8.882, "p95_ms": 7.276}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.977, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 347}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4255.777, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5597ee0c7e80] error while decoding MB 102 43, bytestream -35 [in#0/rtsp @ 0x5597ee046e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5597ee046e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T10:36:06+09:00", "last_error_summary": "[h264 @ 0x5597ee0c7e80] error while decoding MB 102 43, bytestream -35 [in#0/rtsp @ 0x5597ee046e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5597ee046e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T10:32:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 200.332, "last_started_at": "2026-07-02T10:32:46+09:00", "pid": 2374, "process": {"exists": false, "pid": 2374}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 159.599, "sample_count": 2086, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 41655, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T10:36:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.073, "last_frame_pushed_at": "2026-07-02T10:36:06+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-02T10:36:06+09:00", "mjpeg": {"age_sec": 0.054, "exists": true, "mtime": "2026-07-02T10:36:06.310202+09:00", "mtime_epoch": 1782956166.3102024, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 250386}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T10:36:06+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1147.559, "name": "celery", "pid": 267, "rss_mb": 921.578, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 83, "pushed_frames": 83, "read_frames": 83, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.771, "pushed": 2.771, "read": 2.771}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.327, "count": 83, "max_ms": 5.148, "p95_ms": 4.816}, "gstreamer_push": {"avg_ms": 1.2, "count": 83, "max_ms": 2.169, "p95_ms": 1.383}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.869, "count": 83, "max_ms": 75.558, "p95_ms": 69.712}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.895, "count": 83, "max_ms": 100.21, "p95_ms": 87.571}, "yolo11_640_fire_11class": {"avg_ms": 57.683, "count": 83, "max_ms": 79.685, "p95_ms": 71.955}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.814, "count": 83, "max_ms": 96.119, "p95_ms": 82.067}}, "truncated_count": 0}, "processing_total": {"avg_ms": 100.479, "count": 83, "max_ms": 152.38, "p95_ms": 137.903}, "read_interval": {"avg_ms": 361.443, "count": 83, "max_ms": 11958.58, "p95_ms": 154.151}, "triton_wait": {"avg_ms": 65.711, "count": 83, "max_ms": 100.223, "p95_ms": 87.583}, "visualize": {"avg_ms": 4.517, "count": 83, "max_ms": 6.54, "p95_ms": 5.469}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.954, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 347}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4255.406, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T10:36:06+09:00", "last_error_summary": "[h264 @ 0x5597ee0c7e80] error while decoding MB 102 43, bytestream -35 [in#0/rtsp @ 0x5597ee046e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5597ee046e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T10:36:06+09:00", "last_error_summary": "[h264 @ 0x5597ee0c7e80] error while decoding MB 102 43, bytestream -35 [in#0/rtsp @ 0x5597ee046e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5597ee046e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T10:36:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 200.382, "last_started_at": "2026-07-02T10:32:46+09:00", "pid": 2374, "process": {"exists": false, "pid": 2374}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 159.599, "sample_count": 2086, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 41656, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T10:36:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-02T10:36:06+09:00", "last_frame_read_age_sec": 0.111, "last_frame_read_at": "2026-07-02T10:36:06+09:00", "mjpeg": {"age_sec": 0.103, "exists": true, "mtime": "2026-07-02T10:36:06.310202+09:00", "mtime_epoch": 1782956166.3102024, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 250386}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T10:36:06+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1147.559, "name": "celery", "pid": 267, "rss_mb": 921.586, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 83, "pushed_frames": 83, "read_frames": 83, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.768, "pushed": 2.768, "read": 2.768}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.333, "count": 83, "max_ms": 5.148, "p95_ms": 4.816}, "gstreamer_push": {"avg_ms": 1.202, "count": 83, "max_ms": 2.169, "p95_ms": 1.395}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.889, "count": 83, "max_ms": 75.558, "p95_ms": 69.712}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.507, "count": 83, "max_ms": 100.21, "p95_ms": 87.407}, "yolo11_640_fire_11class": {"avg_ms": 57.643, "count": 83, "max_ms": 79.685, "p95_ms": 71.955}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.721, "count": 83, "max_ms": 88.409, "p95_ms": 82.067}}, "truncated_count": 0}, "processing_total": {"avg_ms": 100.387, "count": 83, "max_ms": 152.38, "p95_ms": 137.903}, "read_interval": {"avg_ms": 361.443, "count": 83, "max_ms": 11958.58, "p95_ms": 154.151}, "triton_wait": {"avg_ms": 65.609, "count": 83, "max_ms": 100.223, "p95_ms": 87.583}, "visualize": {"avg_ms": 4.518, "count": 83, "max_ms": 6.54, "p95_ms": 5.469}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.985, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 347}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4255.41, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T10:36:06+09:00", "last_error_summary": "[h264 @ 0x5597ee0c7e80] error while decoding MB 102 43, bytestream -35 [in#0/rtsp @ 0x5597ee046e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5597ee046e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T10:36:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 200.389, "last_started_at": "2026-07-02T10:32:46+09:00", "pid": 2374, "process": {"exists": false, "pid": 2374}, "restart_count": 3, "restart_count_per_hour": 3, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 4}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 159.599, "sample_count": 2086, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 41656, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T10:36:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-02T10:36:06+09:00", "last_frame_read_age_sec": 0.118, "last_frame_read_at": "2026-07-02T10:36:06+09:00", "mjpeg": {"age_sec": 0.111, "exists": true, "mtime": "2026-07-02T10:36:06.310202+09:00", "mtime_epoch": 1782956166.3102024, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 250386}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T10:36:06+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1147.559, "name": "celery", "pid": 267, "rss_mb": 921.586, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 83, "pushed_frames": 83, "read_frames": 83, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.767, "pushed": 2.767, "read": 2.767}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.333, "count": 83, "max_ms": 5.148, "p95_ms": 4.816}, "gstreamer_push": {"avg_ms": 1.202, "count": 83, "max_ms": 2.169, "p95_ms": 1.395}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.889, "count": 83, "max_ms": 75.558, "p95_ms": 69.712}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.507, "count": 83, "max_ms": 100.21, "p95_ms": 87.407}, "yolo11_640_fire_11class": {"avg_ms": 57.643, "count": 83, "max_ms": 79.685, "p95_ms": 71.955}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.721, "count": 83, "max_ms": 88.409, "p95_ms": 82.067}}, "truncated_count": 0}, "processing_total": {"avg_ms": 100.387, "count": 83, "max_ms": 152.38, "p95_ms": 137.903}, "read_interval": {"avg_ms": 361.443, "count": 83, "max_ms": 11958.58, "p95_ms": 154.151}, "triton_wait": {"avg_ms": 65.609, "count": 83, "max_ms": 100.223, "p95_ms": 87.583}, "visualize": {"avg_ms": 4.518, "count": 83, "max_ms": 6.54, "p95_ms": 5.469}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 347}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4268.91, "limit_mb": 8796093022207.996}, "detail": {"pid": 2526, "restart_count": 4, "start_count": 5}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 4, "h264_decode_error_count": 4, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T10:36:06+09:00", "last_error_summary": "[h264 @ 0x5597ee0c7e80] error while decoding MB 102 43, bytestream -35 [in#0/rtsp @ 0x5597ee046e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5597ee046e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T10:36:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T10:36:08+09:00", "pid": 2526, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.469, "name": "ffmpeg", "pid": 2526, "rss_mb": 58.469, "state": "S (sleeping)", "threads": 1}, "restart_count": 4, "restart_count_per_hour": 4, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 159.735, "sample_count": 2051, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 41656, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T10:36:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 4.48, "last_frame_pushed_at": "2026-07-02T10:36:06+09:00", "last_frame_read_age_sec": 4.582, "last_frame_read_at": "2026-07-02T10:36:06+09:00", "mjpeg": {"age_sec": 4.463, "exists": true, "mtime": "2026-07-02T10:36:06.422203+09:00", "mtime_epoch": 1782956166.4222033, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 250342}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T10:36:10+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1147.559, "name": "celery", "pid": 267, "rss_mb": 921.586, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 42, "pushed_frames": 42, "read_frames": 41, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.406, "pushed": 1.406, "read": 1.373}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.653, "count": 42, "max_ms": 5.148, "p95_ms": 4.822}, "gstreamer_push": {"avg_ms": 1.161, "count": 42, "max_ms": 2.169, "p95_ms": 1.379}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.437, "count": 42, "max_ms": 75.558, "p95_ms": 73.985}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.518, "count": 42, "max_ms": 100.21, "p95_ms": 87.571}, "yolo11_640_fire_11class": {"avg_ms": 57.054, "count": 42, "max_ms": 79.356, "p95_ms": 72.16}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.487, "count": 42, "max_ms": 88.409, "p95_ms": 83.15}}, "truncated_count": 0}, "processing_total": {"avg_ms": 106.166, "count": 42, "max_ms": 152.38, "p95_ms": 142.35}, "read_interval": {"avg_ms": 620.182, "count": 41, "max_ms": 11958.58, "p95_ms": 164.729}, "triton_wait": {"avg_ms": 66.104, "count": 42, "max_ms": 100.223, "p95_ms": 88.422}, "visualize": {"avg_ms": 4.667, "count": 42, "max_ms": 6.54, "p95_ms": 5.62}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.868, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 400}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4272.258, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x564564618880] error while decoding MB 17 64, bytestream -29", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T10:50:06+09:00", "last_error_summary": "[h264 @ 0x564564618880] error while decoding MB 17 64, bytestream -29", "last_exit_at": "2026-07-02T10:36:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 837.623, "last_started_at": "2026-07-02T10:36:08+09:00", "pid": 2526, "process": {"exists": false, "pid": 2526}, "restart_count": 4, "restart_count_per_hour": 2, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.683, "sample_count": 2308, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 48034, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T10:50:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.068, "last_frame_pushed_at": "2026-07-02T10:50:06+09:00", "last_frame_read_age_sec": 0.058, "last_frame_read_at": "2026-07-02T10:50:06+09:00", "mjpeg": {"age_sec": 0.049, "exists": true, "mtime": "2026-07-02T10:50:06.458280+09:00", "mtime_epoch": 1782957006.4582803, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 230716}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T10:50:06+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1147.559, "name": "celery", "pid": 267, "rss_mb": 924.422, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 228, "pushed_frames": 228, "read_frames": 228, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.624, "pushed": 7.624, "read": 7.624}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.388, "count": 228, "max_ms": 6.302, "p95_ms": 5.018}, "gstreamer_push": {"avg_ms": 1.125, "count": 228, "max_ms": 1.33, "p95_ms": 1.223}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.167, "count": 228, "max_ms": 78.871, "p95_ms": 67.726}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.502, "count": 228, "max_ms": 108.68, "p95_ms": 90.275}, "yolo11_640_fire_11class": {"avg_ms": 56.563, "count": 228, "max_ms": 86.016, "p95_ms": 71.392}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.395, "count": 228, "max_ms": 93.856, "p95_ms": 82.279}}, "truncated_count": 0}, "processing_total": {"avg_ms": 118.365, "count": 228, "max_ms": 168.594, "p95_ms": 146.843}, "read_interval": {"avg_ms": 131.376, "count": 228, "max_ms": 183.315, "p95_ms": 160.074}, "triton_wait": {"avg_ms": 64.971, "count": 228, "max_ms": 108.694, "p95_ms": 90.601}, "visualize": {"avg_ms": 5.202, "count": 228, "max_ms": 6.853, "p95_ms": 5.876}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.904, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 400}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4265.445, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T10:50:06+09:00", "last_error_summary": "[h264 @ 0x564564618880] error while decoding MB 17 64, bytestream -29", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T10:50:06+09:00", "last_error_summary": "[h264 @ 0x564564618880] error while decoding MB 17 64, bytestream -29", "last_exit_at": "2026-07-02T10:50:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 837.708, "last_started_at": "2026-07-02T10:36:08+09:00", "pid": 2526, "process": {"exists": false, "pid": 2526}, "restart_count": 4, "restart_count_per_hour": 2, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.683, "sample_count": 2309, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 48035, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T10:50:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T10:50:06+09:00", "last_frame_read_age_sec": 0.142, "last_frame_read_at": "2026-07-02T10:50:06+09:00", "mjpeg": {"age_sec": 0.133, "exists": true, "mtime": "2026-07-02T10:50:06.458280+09:00", "mtime_epoch": 1782957006.4582803, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 230716}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T10:50:06+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1147.559, "name": "celery", "pid": 267, "rss_mb": 924.422, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 229, "pushed_frames": 229, "read_frames": 228, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.636, "pushed": 7.636, "read": 7.603}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.389, "count": 228, "max_ms": 6.302, "p95_ms": 5.018}, "gstreamer_push": {"avg_ms": 1.125, "count": 229, "max_ms": 1.33, "p95_ms": 1.223}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.223, "count": 228, "max_ms": 78.871, "p95_ms": 67.726}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.624, "count": 228, "max_ms": 108.68, "p95_ms": 90.275}, "yolo11_640_fire_11class": {"avg_ms": 56.557, "count": 228, "max_ms": 86.016, "p95_ms": 71.392}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.51, "count": 228, "max_ms": 93.856, "p95_ms": 82.279}}, "truncated_count": 0}, "processing_total": {"avg_ms": 118.439, "count": 229, "max_ms": 168.594, "p95_ms": 146.843}, "read_interval": {"avg_ms": 131.376, "count": 228, "max_ms": 183.315, "p95_ms": 160.074}, "triton_wait": {"avg_ms": 65.089, "count": 228, "max_ms": 108.694, "p95_ms": 90.601}, "visualize": {"avg_ms": 5.202, "count": 228, "max_ms": 6.853, "p95_ms": 5.876}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.989, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 400}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4265.559, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T10:50:06+09:00", "last_error_summary": "[h264 @ 0x564564618880] error while decoding MB 17 64, bytestream -29", "last_exit_at": "2026-07-02T10:50:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 837.714, "last_started_at": "2026-07-02T10:36:08+09:00", "pid": 2526, "process": {"exists": false, "pid": 2526}, "restart_count": 4, "restart_count_per_hour": 2, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 5}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.683, "sample_count": 2309, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 48035, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T10:50:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-02T10:50:06+09:00", "last_frame_read_age_sec": 0.148, "last_frame_read_at": "2026-07-02T10:50:06+09:00", "mjpeg": {"age_sec": 0.14, "exists": true, "mtime": "2026-07-02T10:50:06.458280+09:00", "mtime_epoch": 1782957006.4582803, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 230716}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T10:50:06+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1147.559, "name": "celery", "pid": 267, "rss_mb": 924.422, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 229, "pushed_frames": 229, "read_frames": 228, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.635, "pushed": 7.635, "read": 7.601}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.389, "count": 228, "max_ms": 6.302, "p95_ms": 5.018}, "gstreamer_push": {"avg_ms": 1.125, "count": 229, "max_ms": 1.33, "p95_ms": 1.223}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.223, "count": 228, "max_ms": 78.871, "p95_ms": 67.726}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.624, "count": 228, "max_ms": 108.68, "p95_ms": 90.275}, "yolo11_640_fire_11class": {"avg_ms": 56.557, "count": 228, "max_ms": 86.016, "p95_ms": 71.392}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.51, "count": 228, "max_ms": 93.856, "p95_ms": 82.279}}, "truncated_count": 0}, "processing_total": {"avg_ms": 118.439, "count": 229, "max_ms": 168.594, "p95_ms": 146.843}, "read_interval": {"avg_ms": 131.376, "count": 228, "max_ms": 183.315, "p95_ms": 160.074}, "triton_wait": {"avg_ms": 65.089, "count": 228, "max_ms": 108.694, "p95_ms": 90.601}, "visualize": {"avg_ms": 5.202, "count": 228, "max_ms": 6.853, "p95_ms": 5.876}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 400}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4290.492, "limit_mb": 8796093022207.996}, "detail": {"pid": 2678, "restart_count": 5, "start_count": 6}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 5, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T10:50:06+09:00", "last_error_summary": "[h264 @ 0x564564618880] error while decoding MB 17 64, bytestream -29", "last_exit_at": "2026-07-02T10:50:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T10:50:08+09:00", "pid": 2678, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.609, "name": "ffmpeg", "pid": 2678, "rss_mb": 57.609, "state": "S (sleeping)", "threads": 1}, "restart_count": 5, "restart_count_per_hour": 3, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.683, "sample_count": 2276, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 48035, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T10:50:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 4.37, "last_frame_pushed_at": "2026-07-02T10:50:06+09:00", "last_frame_read_age_sec": 4.507, "last_frame_read_at": "2026-07-02T10:50:06+09:00", "mjpeg": {"age_sec": 4.351, "exists": true, "mtime": "2026-07-02T10:50:06.606300+09:00", "mtime_epoch": 1782957006.6063, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 236646}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T10:50:10+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1147.559, "name": "celery", "pid": 267, "rss_mb": 924.422, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 194, "pushed_frames": 195, "read_frames": 194, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.467, "pushed": 6.5, "read": 6.467}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.403, "count": 194, "max_ms": 6.302, "p95_ms": 5.018}, "gstreamer_push": {"avg_ms": 1.132, "count": 195, "max_ms": 1.33, "p95_ms": 1.223}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.402, "count": 194, "max_ms": 78.871, "p95_ms": 67.726}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.243, "count": 194, "max_ms": 108.68, "p95_ms": 91.727}, "yolo11_640_fire_11class": {"avg_ms": 56.575, "count": 194, "max_ms": 82.082, "p95_ms": 71.775}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.559, "count": 194, "max_ms": 93.856, "p95_ms": 82.279}}, "truncated_count": 0}, "processing_total": {"avg_ms": 119.049, "count": 194, "max_ms": 168.594, "p95_ms": 147.078}, "read_interval": {"avg_ms": 132.0, "count": 194, "max_ms": 183.315, "p95_ms": 160.91}, "triton_wait": {"avg_ms": 65.584, "count": 194, "max_ms": 108.694, "p95_ms": 93.014}, "visualize": {"avg_ms": 5.214, "count": 194, "max_ms": 6.853, "p95_ms": 5.929}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 459}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4291.066, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x561b6c2cdf80] error while decoding MB 21 33, bytestream -21", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 6, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:05:37+09:00", "last_error_summary": "[h264 @ 0x561b6c2cdf80] error while decoding MB 21 33, bytestream -21", "last_exit_at": "2026-07-02T10:50:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 928.529, "last_started_at": "2026-07-02T10:50:08+09:00", "pid": 2678, "process": {"exists": false, "pid": 2678}, "restart_count": 5, "restart_count_per_hour": 3, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 142.005, "sample_count": 2322, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 55180, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:05:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.064, "last_frame_pushed_at": "2026-07-02T11:05:37+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-02T11:05:37+09:00", "mjpeg": {"age_sec": 0.048, "exists": true, "mtime": "2026-07-02T11:05:37.437097+09:00", "mtime_epoch": 1782957937.4370966, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 220894}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T11:05:37+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1147.559, "name": "celery", "pid": 267, "rss_mb": 927.773, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 235, "pushed_frames": 235, "read_frames": 235, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.839, "pushed": 7.839, "read": 7.839}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.689, "count": 234, "max_ms": 6.994, "p95_ms": 5.962}, "gstreamer_push": {"avg_ms": 1.118, "count": 235, "max_ms": 1.562, "p95_ms": 1.232}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.272, "count": 234, "max_ms": 75.591, "p95_ms": 67.286}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.642, "count": 234, "max_ms": 93.717, "p95_ms": 82.062}, "yolo11_640_fire_11class": {"avg_ms": 55.056, "count": 234, "max_ms": 77.322, "p95_ms": 68.899}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.794, "count": 234, "max_ms": 90.468, "p95_ms": 79.661}}, "truncated_count": 0}, "processing_total": {"avg_ms": 114.977, "count": 235, "max_ms": 155.047, "p95_ms": 138.608}, "read_interval": {"avg_ms": 127.896, "count": 235, "max_ms": 169.208, "p95_ms": 150.662}, "triton_wait": {"avg_ms": 62.078, "count": 234, "max_ms": 93.732, "p95_ms": 83.37}, "visualize": {"avg_ms": 5.283, "count": 234, "max_ms": 9.91, "p95_ms": 6.224}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.978, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 459}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4290.672, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T11:05:37+09:00", "last_error_summary": "[h264 @ 0x561b6c2cdf80] error while decoding MB 21 33, bytestream -21", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 6, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:05:37+09:00", "last_error_summary": "[h264 @ 0x561b6c2cdf80] error while decoding MB 21 33, bytestream -21", "last_exit_at": "2026-07-02T11:05:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 928.595, "last_started_at": "2026-07-02T10:50:08+09:00", "pid": 2678, "process": {"exists": false, "pid": 2678}, "restart_count": 5, "restart_count_per_hour": 3, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 142.005, "sample_count": 2323, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 55181, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:05:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T11:05:37+09:00", "last_frame_read_age_sec": 0.119, "last_frame_read_at": "2026-07-02T11:05:37+09:00", "mjpeg": {"age_sec": 0.114, "exists": true, "mtime": "2026-07-02T11:05:37.437097+09:00", "mtime_epoch": 1782957937.4370966, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 220894}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T11:05:37+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1147.559, "name": "celery", "pid": 267, "rss_mb": 927.773, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 235, "pushed_frames": 235, "read_frames": 234, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.854, "pushed": 7.854, "read": 7.821}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.704, "count": 235, "max_ms": 8.314, "p95_ms": 6.302}, "gstreamer_push": {"avg_ms": 1.118, "count": 235, "max_ms": 1.562, "p95_ms": 1.232}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.239, "count": 235, "max_ms": 75.591, "p95_ms": 67.286}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.627, "count": 235, "max_ms": 93.717, "p95_ms": 82.062}, "yolo11_640_fire_11class": {"avg_ms": 55.033, "count": 235, "max_ms": 77.322, "p95_ms": 68.899}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.757, "count": 235, "max_ms": 90.468, "p95_ms": 79.661}}, "truncated_count": 0}, "processing_total": {"avg_ms": 114.896, "count": 235, "max_ms": 155.047, "p95_ms": 138.608}, "read_interval": {"avg_ms": 127.829, "count": 234, "max_ms": 169.208, "p95_ms": 150.662}, "triton_wait": {"avg_ms": 62.036, "count": 235, "max_ms": 93.732, "p95_ms": 83.37}, "visualize": {"avg_ms": 5.288, "count": 235, "max_ms": 9.91, "p95_ms": 6.471}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.921, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 459}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4290.73, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 6, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:05:37+09:00", "last_error_summary": "[h264 @ 0x561b6c2cdf80] error while decoding MB 21 33, bytestream -21", "last_exit_at": "2026-07-02T11:05:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 928.601, "last_started_at": "2026-07-02T10:50:08+09:00", "pid": 2678, "process": {"exists": false, "pid": 2678}, "restart_count": 5, "restart_count_per_hour": 3, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 6}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 142.005, "sample_count": 2323, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 55181, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:05:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-02T11:05:37+09:00", "last_frame_read_age_sec": 0.125, "last_frame_read_at": "2026-07-02T11:05:37+09:00", "mjpeg": {"age_sec": 0.12, "exists": true, "mtime": "2026-07-02T11:05:37.437097+09:00", "mtime_epoch": 1782957937.4370966, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 220894}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T11:05:37+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1147.559, "name": "celery", "pid": 267, "rss_mb": 927.773, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 235, "pushed_frames": 235, "read_frames": 234, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.852, "pushed": 7.852, "read": 7.819}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.704, "count": 235, "max_ms": 8.314, "p95_ms": 6.302}, "gstreamer_push": {"avg_ms": 1.118, "count": 235, "max_ms": 1.562, "p95_ms": 1.232}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.239, "count": 235, "max_ms": 75.591, "p95_ms": 67.286}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.627, "count": 235, "max_ms": 93.717, "p95_ms": 82.062}, "yolo11_640_fire_11class": {"avg_ms": 55.033, "count": 235, "max_ms": 77.322, "p95_ms": 68.899}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.757, "count": 235, "max_ms": 90.468, "p95_ms": 79.661}}, "truncated_count": 0}, "processing_total": {"avg_ms": 114.896, "count": 235, "max_ms": 155.047, "p95_ms": 138.608}, "read_interval": {"avg_ms": 127.829, "count": 234, "max_ms": 169.208, "p95_ms": 150.662}, "triton_wait": {"avg_ms": 62.036, "count": 235, "max_ms": 93.732, "p95_ms": 83.37}, "visualize": {"avg_ms": 5.288, "count": 235, "max_ms": 9.91, "p95_ms": 6.471}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.927, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 459}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4311.914, "limit_mb": 8796093022207.996}, "detail": {"pid": 2830, "restart_count": 6, "start_count": 7}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 6, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:05:37+09:00", "last_error_summary": "[h264 @ 0x561b6c2cdf80] error while decoding MB 21 33, bytestream -21", "last_exit_at": "2026-07-02T11:05:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T11:05:39+09:00", "pid": 2830, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.504, "name": "ffmpeg", "pid": 2830, "rss_mb": 58.504, "state": "S (sleeping)", "threads": 1}, "restart_count": 6, "restart_count_per_hour": 4, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 142.005, "sample_count": 2291, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 55181, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:05:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 4.114, "last_frame_pushed_at": "2026-07-02T11:05:37+09:00", "last_frame_read_age_sec": 4.228, "last_frame_read_at": "2026-07-02T11:05:37+09:00", "mjpeg": {"age_sec": 4.099, "exists": true, "mtime": "2026-07-02T11:05:37.561102+09:00", "mtime_epoch": 1782957937.5611022, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 221375}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T11:05:41+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1147.559, "name": "celery", "pid": 267, "rss_mb": 927.773, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 203, "pushed_frames": 203, "read_frames": 202, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.774, "pushed": 6.774, "read": 6.741}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.68, "count": 202, "max_ms": 8.314, "p95_ms": 5.58}, "gstreamer_push": {"avg_ms": 1.122, "count": 203, "max_ms": 1.562, "p95_ms": 1.232}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.291, "count": 202, "max_ms": 74.25, "p95_ms": 67.286}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.966, "count": 202, "max_ms": 93.717, "p95_ms": 82.062}, "yolo11_640_fire_11class": {"avg_ms": 54.96, "count": 202, "max_ms": 75.169, "p95_ms": 68.776}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.9, "count": 202, "max_ms": 88.451, "p95_ms": 79.661}}, "truncated_count": 0}, "processing_total": {"avg_ms": 114.97, "count": 203, "max_ms": 150.529, "p95_ms": 136.784}, "read_interval": {"avg_ms": 127.876, "count": 202, "max_ms": 163.048, "p95_ms": 150.662}, "triton_wait": {"avg_ms": 62.324, "count": 202, "max_ms": 93.732, "p95_ms": 83.37}, "visualize": {"avg_ms": 5.232, "count": 202, "max_ms": 9.91, "p95_ms": 5.766}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.968, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 494}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4302.332, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x557e3c3015c0] error while decoding MB 37 41, bytestream -5 [in#0/rtsp @ 0x557e3c219e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557e3c219e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 7, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:15:14+09:00", "last_error_summary": "[h264 @ 0x557e3c3015c0] error while decoding MB 37 41, bytestream -5 [in#0/rtsp @ 0x557e3c219e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557e3c219e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T11:05:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 574.797, "last_started_at": "2026-07-02T11:05:39+09:00", "pid": 2830, "process": {"exists": false, "pid": 2830}, "restart_count": 6, "restart_count_per_hour": 4, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.668, "sample_count": 2147, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 59379, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:15:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-02T11:15:14+09:00", "last_frame_read_age_sec": 0.067, "last_frame_read_at": "2026-07-02T11:15:14+09:00", "mjpeg": {"age_sec": 0.054, "exists": true, "mtime": "2026-07-02T11:15:14.402386+09:00", "mtime_epoch": 1782958514.4023857, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 230795}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T11:15:14+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1147.559, "name": "celery", "pid": 267, "rss_mb": 929.836, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 86, "pushed_frames": 86, "read_frames": 86, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.878, "pushed": 2.878, "read": 2.878}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.59, "count": 86, "max_ms": 5.448, "p95_ms": 5.157}, "gstreamer_push": {"avg_ms": 1.131, "count": 86, "max_ms": 1.268, "p95_ms": 1.252}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.292, "count": 86, "max_ms": 75.022, "p95_ms": 66.995}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.803, "count": 86, "max_ms": 101.048, "p95_ms": 92.792}, "yolo11_640_fire_11class": {"avg_ms": 54.902, "count": 86, "max_ms": 78.381, "p95_ms": 69.806}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.67, "count": 86, "max_ms": 92.213, "p95_ms": 80.625}}, "truncated_count": 0}, "processing_total": {"avg_ms": 117.834, "count": 86, "max_ms": 152.488, "p95_ms": 145.623}, "read_interval": {"avg_ms": 348.263, "count": 86, "max_ms": 10165.919, "p95_ms": 160.636}, "triton_wait": {"avg_ms": 65.368, "count": 86, "max_ms": 101.061, "p95_ms": 92.803}, "visualize": {"avg_ms": 5.205, "count": 86, "max_ms": 7.154, "p95_ms": 5.521}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.883, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 494}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4301.766, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T11:15:14+09:00", "last_error_summary": "[h264 @ 0x557e3c3015c0] error while decoding MB 37 41, bytestream -5 [in#0/rtsp @ 0x557e3c219e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557e3c219e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 7, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:15:14+09:00", "last_error_summary": "[h264 @ 0x557e3c3015c0] error while decoding MB 37 41, bytestream -5 [in#0/rtsp @ 0x557e3c219e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557e3c219e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T11:15:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 574.867, "last_started_at": "2026-07-02T11:05:39+09:00", "pid": 2830, "process": {"exists": false, "pid": 2830}, "restart_count": 6, "restart_count_per_hour": 4, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.668, "sample_count": 2147, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 59380, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:15:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T11:15:14+09:00", "last_frame_read_age_sec": 0.136, "last_frame_read_at": "2026-07-02T11:15:14+09:00", "mjpeg": {"age_sec": 0.123, "exists": true, "mtime": "2026-07-02T11:15:14.402386+09:00", "mtime_epoch": 1782958514.4023857, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 230795}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T11:15:14+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1147.559, "name": "celery", "pid": 267, "rss_mb": 929.836, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 87, "pushed_frames": 87, "read_frames": 86, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.905, "pushed": 2.905, "read": 2.871}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.591, "count": 87, "max_ms": 5.448, "p95_ms": 5.157}, "gstreamer_push": {"avg_ms": 1.131, "count": 87, "max_ms": 1.268, "p95_ms": 1.252}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.516, "count": 87, "max_ms": 75.022, "p95_ms": 69.037}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.624, "count": 87, "max_ms": 101.048, "p95_ms": 92.792}, "yolo11_640_fire_11class": {"avg_ms": 55.152, "count": 87, "max_ms": 78.381, "p95_ms": 70.184}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.847, "count": 87, "max_ms": 92.213, "p95_ms": 80.625}}, "truncated_count": 0}, "processing_total": {"avg_ms": 117.973, "count": 87, "max_ms": 152.488, "p95_ms": 145.623}, "read_interval": {"avg_ms": 348.263, "count": 86, "max_ms": 10165.919, "p95_ms": 160.636}, "triton_wait": {"avg_ms": 65.498, "count": 87, "max_ms": 101.061, "p95_ms": 92.803}, "visualize": {"avg_ms": 5.206, "count": 87, "max_ms": 7.154, "p95_ms": 5.521}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.952, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 494}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4301.676, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 7, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:15:14+09:00", "last_error_summary": "[h264 @ 0x557e3c3015c0] error while decoding MB 37 41, bytestream -5 [in#0/rtsp @ 0x557e3c219e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557e3c219e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T11:15:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 574.872, "last_started_at": "2026-07-02T11:05:39+09:00", "pid": 2830, "process": {"exists": false, "pid": 2830}, "restart_count": 6, "restart_count_per_hour": 4, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 7}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.668, "sample_count": 2147, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 59380, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:15:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-02T11:15:14+09:00", "last_frame_read_age_sec": 0.141, "last_frame_read_at": "2026-07-02T11:15:14+09:00", "mjpeg": {"age_sec": 0.128, "exists": true, "mtime": "2026-07-02T11:15:14.402386+09:00", "mtime_epoch": 1782958514.4023857, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 230795}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T11:15:14+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1147.559, "name": "celery", "pid": 267, "rss_mb": 929.836, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 87, "pushed_frames": 87, "read_frames": 86, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.904, "pushed": 2.904, "read": 2.871}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.592, "count": 86, "max_ms": 5.448, "p95_ms": 5.157}, "gstreamer_push": {"avg_ms": 1.131, "count": 87, "max_ms": 1.268, "p95_ms": 1.252}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.494, "count": 86, "max_ms": 75.022, "p95_ms": 69.037}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.419, "count": 86, "max_ms": 101.048, "p95_ms": 92.792}, "yolo11_640_fire_11class": {"avg_ms": 55.127, "count": 86, "max_ms": 78.381, "p95_ms": 70.184}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.799, "count": 86, "max_ms": 92.213, "p95_ms": 80.625}}, "truncated_count": 0}, "processing_total": {"avg_ms": 117.973, "count": 87, "max_ms": 152.488, "p95_ms": 145.623}, "read_interval": {"avg_ms": 348.263, "count": 86, "max_ms": 10165.919, "p95_ms": 160.636}, "triton_wait": {"avg_ms": 65.315, "count": 86, "max_ms": 101.061, "p95_ms": 92.803}, "visualize": {"avg_ms": 5.206, "count": 86, "max_ms": 7.154, "p95_ms": 5.521}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.957, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 494}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4469.852, "limit_mb": 8796093022207.996}, "detail": {"pid": 2982, "restart_count": 7, "start_count": 8}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 7, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:15:14+09:00", "last_error_summary": "[h264 @ 0x557e3c3015c0] error while decoding MB 37 41, bytestream -5 [in#0/rtsp @ 0x557e3c219e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557e3c219e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T11:15:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T11:15:17+09:00", "pid": 2982, "process": {"exists": true, "fd_count": 5, "hwm_mb": 197.754, "name": "ffmpeg", "pid": 2982, "rss_mb": 197.754, "state": "S (sleeping)", "threads": 106}, "restart_count": 7, "restart_count_per_hour": 5, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.668, "sample_count": 2109, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 59380, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:15:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 4.929, "last_frame_pushed_at": "2026-07-02T11:15:14+09:00", "last_frame_read_age_sec": 5.061, "last_frame_read_at": "2026-07-02T11:15:14+09:00", "mjpeg": {"age_sec": 4.912, "exists": true, "mtime": "2026-07-02T11:15:14.538390+09:00", "mtime_epoch": 1782958514.53839, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 230701}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T11:15:19+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1147.559, "name": "celery", "pid": 267, "rss_mb": 929.836, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 49, "pushed_frames": 49, "read_frames": 48, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.637, "pushed": 1.637, "read": 1.603}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.606, "count": 49, "max_ms": 5.448, "p95_ms": 5.144}, "gstreamer_push": {"avg_ms": 1.119, "count": 49, "max_ms": 1.263, "p95_ms": 1.234}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.556, "count": 49, "max_ms": 75.022, "p95_ms": 71.852}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.48, "count": 49, "max_ms": 97.748, "p95_ms": 92.792}, "yolo11_640_fire_11class": {"avg_ms": 55.503, "count": 49, "max_ms": 78.381, "p95_ms": 74.35}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.56, "count": 49, "max_ms": 92.213, "p95_ms": 86.208}}, "truncated_count": 0}, "processing_total": {"avg_ms": 118.219, "count": 49, "max_ms": 152.488, "p95_ms": 145.623}, "read_interval": {"avg_ms": 520.423, "count": 48, "max_ms": 10165.919, "p95_ms": 164.022}, "triton_wait": {"avg_ms": 65.777, "count": 49, "max_ms": 97.76, "p95_ms": 92.803}, "visualize": {"avg_ms": 5.203, "count": 49, "max_ms": 5.965, "p95_ms": 5.521}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.936, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 539}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4315.512, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x562b80fe0e80] error while decoding MB 49 18, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 8, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:27:38+09:00", "last_error_summary": "[h264 @ 0x562b80fe0e80] error while decoding MB 49 18, bytestream -9", "last_exit_at": "2026-07-02T11:15:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 741.542, "last_started_at": "2026-07-02T11:15:17+09:00", "pid": 2982, "process": {"exists": false, "pid": 2982}, "restart_count": 7, "restart_count_per_hour": 5, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 169.328, "sample_count": 2052, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 64789, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:27:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.086, "last_frame_pushed_at": "2026-07-02T11:27:38+09:00", "last_frame_read_age_sec": 0.076, "last_frame_read_at": "2026-07-02T11:27:38+09:00", "mjpeg": {"age_sec": 0.071, "exists": true, "mtime": "2026-07-02T11:27:38.920259+09:00", "mtime_epoch": 1782959258.9202595, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 227038}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T11:27:39+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1147.559, "name": "celery", "pid": 267, "rss_mb": 932.414, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 216, "pushed_frames": 216, "read_frames": 216, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.207, "pushed": 7.207, "read": 7.207}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.924, "count": 216, "max_ms": 7.936, "p95_ms": 6.17}, "gstreamer_push": {"avg_ms": 1.105, "count": 216, "max_ms": 1.636, "p95_ms": 1.313}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.696, "count": 216, "max_ms": 79.25, "p95_ms": 72.151}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.058, "count": 216, "max_ms": 110.34, "p95_ms": 96.59}, "yolo11_640_fire_11class": {"avg_ms": 55.078, "count": 216, "max_ms": 82.148, "p95_ms": 72.84}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.6, "count": 216, "max_ms": 100.655, "p95_ms": 89.499}}, "truncated_count": 0}, "processing_total": {"avg_ms": 126.787, "count": 216, "max_ms": 187.831, "p95_ms": 173.517}, "read_interval": {"avg_ms": 139.016, "count": 216, "max_ms": 198.06, "p95_ms": 185.195}, "triton_wait": {"avg_ms": 65.903, "count": 216, "max_ms": 110.352, "p95_ms": 97.095}, "visualize": {"avg_ms": 5.541, "count": 216, "max_ms": 9.19, "p95_ms": 6.875}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.973, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 539}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4315.188, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T11:27:38+09:00", "last_error_summary": "[h264 @ 0x562b80fe0e80] error while decoding MB 49 18, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 8, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:27:38+09:00", "last_error_summary": "[h264 @ 0x562b80fe0e80] error while decoding MB 49 18, bytestream -9", "last_exit_at": "2026-07-02T11:27:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 741.598, "last_started_at": "2026-07-02T11:15:17+09:00", "pid": 2982, "process": {"exists": false, "pid": 2982}, "restart_count": 7, "restart_count_per_hour": 5, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 169.328, "sample_count": 2053, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 64789, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:27:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T11:27:39+09:00", "last_frame_read_age_sec": 0.131, "last_frame_read_at": "2026-07-02T11:27:38+09:00", "mjpeg": {"age_sec": 0.127, "exists": true, "mtime": "2026-07-02T11:27:38.920259+09:00", "mtime_epoch": 1782959258.9202595, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 227038}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T11:27:39+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1147.559, "name": "celery", "pid": 267, "rss_mb": 932.414, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 216, "pushed_frames": 216, "read_frames": 215, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.231, "pushed": 7.231, "read": 7.197}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.924, "count": 216, "max_ms": 7.936, "p95_ms": 6.17}, "gstreamer_push": {"avg_ms": 1.106, "count": 216, "max_ms": 1.636, "p95_ms": 1.313}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.696, "count": 216, "max_ms": 79.25, "p95_ms": 72.151}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.058, "count": 216, "max_ms": 110.34, "p95_ms": 96.59}, "yolo11_640_fire_11class": {"avg_ms": 55.078, "count": 216, "max_ms": 82.148, "p95_ms": 72.84}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.6, "count": 216, "max_ms": 100.655, "p95_ms": 89.499}}, "truncated_count": 0}, "processing_total": {"avg_ms": 126.736, "count": 216, "max_ms": 187.831, "p95_ms": 173.517}, "read_interval": {"avg_ms": 138.999, "count": 215, "max_ms": 198.06, "p95_ms": 185.195}, "triton_wait": {"avg_ms": 65.903, "count": 216, "max_ms": 110.352, "p95_ms": 97.095}, "visualize": {"avg_ms": 5.541, "count": 216, "max_ms": 9.19, "p95_ms": 6.875}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.873, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 539}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4315.18, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 8, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:27:38+09:00", "last_error_summary": "[h264 @ 0x562b80fe0e80] error while decoding MB 49 18, bytestream -9", "last_exit_at": "2026-07-02T11:27:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 741.608, "last_started_at": "2026-07-02T11:15:17+09:00", "pid": 2982, "process": {"exists": false, "pid": 2982}, "restart_count": 7, "restart_count_per_hour": 5, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 169.328, "sample_count": 2052, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 64789, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:27:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.017, "last_frame_pushed_at": "2026-07-02T11:27:39+09:00", "last_frame_read_age_sec": 0.141, "last_frame_read_at": "2026-07-02T11:27:38+09:00", "mjpeg": {"age_sec": 0.137, "exists": true, "mtime": "2026-07-02T11:27:38.920259+09:00", "mtime_epoch": 1782959258.9202595, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 227038}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T11:27:39+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1147.559, "name": "celery", "pid": 267, "rss_mb": 932.414, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 216, "pushed_frames": 216, "read_frames": 215, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.228, "pushed": 7.228, "read": 7.195}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.924, "count": 216, "max_ms": 7.936, "p95_ms": 6.17}, "gstreamer_push": {"avg_ms": 1.106, "count": 216, "max_ms": 1.636, "p95_ms": 1.313}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.696, "count": 216, "max_ms": 79.25, "p95_ms": 72.151}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.058, "count": 216, "max_ms": 110.34, "p95_ms": 96.59}, "yolo11_640_fire_11class": {"avg_ms": 55.078, "count": 216, "max_ms": 82.148, "p95_ms": 72.84}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.6, "count": 216, "max_ms": 100.655, "p95_ms": 89.499}}, "truncated_count": 0}, "processing_total": {"avg_ms": 126.736, "count": 216, "max_ms": 187.831, "p95_ms": 173.517}, "read_interval": {"avg_ms": 138.999, "count": 215, "max_ms": 198.06, "p95_ms": 185.195}, "triton_wait": {"avg_ms": 65.903, "count": 216, "max_ms": 110.352, "p95_ms": 97.095}, "visualize": {"avg_ms": 5.541, "count": 216, "max_ms": 9.19, "p95_ms": 6.875}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.883, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 6, "tracking_summary": 539}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4340.195, "limit_mb": 8796093022207.996}, "detail": {"pid": 3134, "restart_count": 8, "start_count": 9}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 8, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:27:38+09:00", "last_error_summary": "[h264 @ 0x562b80fe0e80] error while decoding MB 49 18, bytestream -9", "last_exit_at": "2026-07-02T11:27:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T11:27:41+09:00", "pid": 3134, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.457, "name": "ffmpeg", "pid": 3134, "rss_mb": 58.457, "state": "S (sleeping)", "threads": 1}, "restart_count": 8, "restart_count_per_hour": 6, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 170.047, "sample_count": 2016, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 64789, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:27:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 4.926, "last_frame_pushed_at": "2026-07-02T11:27:39+09:00", "last_frame_read_age_sec": 5.05, "last_frame_read_at": "2026-07-02T11:27:38+09:00", "mjpeg": {"age_sec": 4.906, "exists": true, "mtime": "2026-07-02T11:27:39.060256+09:00", "mtime_epoch": 1782959259.0602565, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 227038}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T11:27:43+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1147.559, "name": "celery", "pid": 267, "rss_mb": 932.414, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 186, "pushed_frames": 186, "read_frames": 185, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.22, "pushed": 6.22, "read": 6.186}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.805, "count": 186, "max_ms": 7.936, "p95_ms": 6.096}, "gstreamer_push": {"avg_ms": 1.113, "count": 186, "max_ms": 1.636, "p95_ms": 1.316}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.505, "count": 186, "max_ms": 79.25, "p95_ms": 71.788}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.579, "count": 186, "max_ms": 110.34, "p95_ms": 96.136}, "yolo11_640_fire_11class": {"avg_ms": 55.059, "count": 186, "max_ms": 82.148, "p95_ms": 72.84}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.363, "count": 186, "max_ms": 100.655, "p95_ms": 88.454}}, "truncated_count": 0}, "processing_total": {"avg_ms": 122.811, "count": 186, "max_ms": 183.42, "p95_ms": 172.256}, "read_interval": {"avg_ms": 135.288, "count": 185, "max_ms": 194.623, "p95_ms": 184.089}, "triton_wait": {"avg_ms": 65.373, "count": 186, "max_ms": 110.352, "p95_ms": 96.271}, "visualize": {"avg_ms": 5.374, "count": 186, "max_ms": 8.059, "p95_ms": 6.847}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.905, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 606}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4329.191, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T10:08:18+09:00", "last_error_summary": "[h264 @ 0x55ee7aa42ec0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ee7aa42ec0] error while decoding MB 0 39, bytestream 4835 [h264 @ 0x55ee7b491f40] cabac decode of qscale diff failed at 71 44 [h264 @ 0x55ee7b491f40] error while decoding MB 71 44, bytestream 2313 [h264 @ 0x55ee7aa9a640] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ee7aa9a640] error while decoding MB 0 41, bytestream 78355 [h264 @ 0x55ee7b6feac0] left block unavailable for requested int", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 13, "h264_decode_error_count_per_hour": 0, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T10:08:18+09:00", "last_error_summary": "[h264 @ 0x55ee7aa42ec0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ee7aa42ec0] error while decoding MB 0 39, bytestream 4835 [h264 @ 0x55ee7b491f40] cabac decode of qscale diff failed at 71 44 [h264 @ 0x55ee7b491f40] error while decoding MB 71 44, bytestream 2313 [h264 @ 0x55ee7aa9a640] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ee7aa9a640] error while decoding MB 0 41, bytestream 78355 [h264 @ 0x55ee7b6feac0] left block unavailable for requested int", "last_exit_at": "2026-07-02T11:29:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 4857.653, "last_started_at": "2026-07-02T10:08:21+09:00", "pid": 2207, "process": {"exists": false, "pid": 2207}, "restart_count": 7, "restart_count_per_hour": 0, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 141.547, "sample_count": 2469, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 72734, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:29:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T11:29:19+09:00", "last_frame_read_age_sec": 0.119, "last_frame_read_at": "2026-07-02T11:29:19+09:00", "mjpeg": {"age_sec": 0.111, "exists": true, "mtime": "2026-07-02T11:29:19.074494+09:00", "mtime_epoch": 1782959359.074494, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187944}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:29:19+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 904.773, "name": "celery", "pid": 237, "rss_mb": 899.855, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 230, "pushed_frames": 230, "read_frames": 229, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.669, "pushed": 7.669, "read": 7.635}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.12, "count": 229, "max_ms": 6.58, "p95_ms": 5.468}, "gstreamer_push": {"avg_ms": 1.166, "count": 230, "max_ms": 4.98, "p95_ms": 2.311}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.028, "count": 229, "max_ms": 74.106, "p95_ms": 64.257}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.913, "count": 229, "max_ms": 99.744, "p95_ms": 85.448}, "yolo11_640_fire_11class": {"avg_ms": 49.91, "count": 229, "max_ms": 76.565, "p95_ms": 66.983}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.212, "count": 229, "max_ms": 96.86, "p95_ms": 84.007}}, "truncated_count": 0}, "processing_total": {"avg_ms": 118.289, "count": 230, "max_ms": 176.792, "p95_ms": 155.095}, "read_interval": {"avg_ms": 130.892, "count": 229, "max_ms": 189.102, "p95_ms": 167.752}, "triton_wait": {"avg_ms": 63.065, "count": 229, "max_ms": 99.756, "p95_ms": 87.287}, "visualize": {"avg_ms": 5.385, "count": 229, "max_ms": 7.807, "p95_ms": 6.56}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 606}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4329.086, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 13, "h264_decode_error_count_per_hour": 0, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T10:08:18+09:00", "last_error_summary": "[h264 @ 0x55ee7aa42ec0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ee7aa42ec0] error while decoding MB 0 39, bytestream 4835 [h264 @ 0x55ee7b491f40] cabac decode of qscale diff failed at 71 44 [h264 @ 0x55ee7b491f40] error while decoding MB 71 44, bytestream 2313 [h264 @ 0x55ee7aa9a640] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ee7aa9a640] error while decoding MB 0 41, bytestream 78355 [h264 @ 0x55ee7b6feac0] left block unavailable for requested int", "last_exit_at": "2026-07-02T11:29:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 4857.659, "last_started_at": "2026-07-02T10:08:21+09:00", "pid": 2207, "process": {"exists": false, "pid": 2207}, "restart_count": 7, "restart_count_per_hour": 0, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": false, "start_count": 8}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 141.547, "sample_count": 2469, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 72734, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:29:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-02T11:29:19+09:00", "last_frame_read_age_sec": 0.124, "last_frame_read_at": "2026-07-02T11:29:19+09:00", "mjpeg": {"age_sec": 0.117, "exists": true, "mtime": "2026-07-02T11:29:19.074494+09:00", "mtime_epoch": 1782959359.074494, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 187944}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:29:19+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 904.773, "name": "celery", "pid": 237, "rss_mb": 899.855, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 230, "pushed_frames": 230, "read_frames": 229, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.667, "pushed": 7.667, "read": 7.634}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.12, "count": 229, "max_ms": 6.58, "p95_ms": 5.468}, "gstreamer_push": {"avg_ms": 1.166, "count": 230, "max_ms": 4.98, "p95_ms": 2.311}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.028, "count": 229, "max_ms": 74.106, "p95_ms": 64.257}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.913, "count": 229, "max_ms": 99.744, "p95_ms": 85.448}, "yolo11_640_fire_11class": {"avg_ms": 49.91, "count": 229, "max_ms": 76.565, "p95_ms": 66.983}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.212, "count": 229, "max_ms": 96.86, "p95_ms": 84.007}}, "truncated_count": 0}, "processing_total": {"avg_ms": 118.289, "count": 230, "max_ms": 176.792, "p95_ms": 155.095}, "read_interval": {"avg_ms": 130.892, "count": 229, "max_ms": 189.102, "p95_ms": 167.752}, "triton_wait": {"avg_ms": 63.065, "count": 229, "max_ms": 99.756, "p95_ms": 87.287}, "visualize": {"avg_ms": 5.385, "count": 229, "max_ms": 7.807, "p95_ms": 6.56}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 606}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4434.129, "limit_mb": 8796093022207.996}, "detail": {"pid": 3286, "restart_count": 8, "start_count": 9}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 5, "h264_decode_error_count": 13, "h264_decode_error_count_per_hour": 0, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T10:08:18+09:00", "last_error_summary": "[h264 @ 0x55ee7aa42ec0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ee7aa42ec0] error while decoding MB 0 39, bytestream 4835 [h264 @ 0x55ee7b491f40] cabac decode of qscale diff failed at 71 44 [h264 @ 0x55ee7b491f40] error while decoding MB 71 44, bytestream 2313 [h264 @ 0x55ee7aa9a640] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55ee7aa9a640] error while decoding MB 0 41, bytestream 78355 [h264 @ 0x55ee7b6feac0] left block unavailable for requested int", "last_exit_at": "2026-07-02T11:29:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T11:29:21+09:00", "pid": 3286, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.707, "name": "ffmpeg", "pid": 3286, "rss_mb": 136.707, "state": "S (sleeping)", "threads": 106}, "restart_count": 8, "restart_count_per_hour": 1, "rtsp_timeout_count": 2, "rtsp_timeout_count_per_hour": 0, "rtsp_tolerant_mode": false, "running": true, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 141.632, "sample_count": 2429, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 72734, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:29:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.041, "last_frame_pushed_at": "2026-07-02T11:29:19+09:00", "last_frame_read_age_sec": 4.154, "last_frame_read_at": "2026-07-02T11:29:19+09:00", "mjpeg": {"age_sec": 4.023, "exists": true, "mtime": "2026-07-02T11:29:19.198492+09:00", "mtime_epoch": 1782959359.1984923, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188685}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:29:23+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 904.773, "name": "celery", "pid": 237, "rss_mb": 899.855, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 197, "pushed_frames": 197, "read_frames": 197, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.567, "pushed": 6.567, "read": 6.567}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.176, "count": 197, "max_ms": 6.58, "p95_ms": 5.493}, "gstreamer_push": {"avg_ms": 1.159, "count": 197, "max_ms": 4.98, "p95_ms": 2.311}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.125, "count": 197, "max_ms": 74.106, "p95_ms": 64.93}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.826, "count": 197, "max_ms": 97.575, "p95_ms": 85.549}, "yolo11_640_fire_11class": {"avg_ms": 50.151, "count": 197, "max_ms": 76.565, "p95_ms": 67.787}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.812, "count": 197, "max_ms": 96.86, "p95_ms": 84.189}}, "truncated_count": 0}, "processing_total": {"avg_ms": 119.292, "count": 197, "max_ms": 176.792, "p95_ms": 158.46}, "read_interval": {"avg_ms": 131.785, "count": 197, "max_ms": 189.102, "p95_ms": 168.948}, "triton_wait": {"avg_ms": 63.281, "count": 197, "max_ms": 97.587, "p95_ms": 88.033}, "visualize": {"avg_ms": 5.407, "count": 197, "max_ms": 7.807, "p95_ms": 6.649}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 614}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4420.152, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5623a5159080] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5623a5159080] error while decoding MB 0 29, bytestream 4610 [h264 @ 0x5623a5b71bc0] top block unavailable for requested intra mode -1 [h264 @ 0x5623a5b71bc0] error while decoding MB 12 0, bytestream 1358 [in#0/rtsp @ 0x5623a501ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5623a501ee00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 15, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:32:23+09:00", "last_error_summary": "[h264 @ 0x5623a5159080] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5623a5159080] error while decoding MB 0 29, bytestream 4610 [h264 @ 0x5623a5b71bc0] top block unavailable for requested intra mode -1 [h264 @ 0x5623a5b71bc0] error while decoding MB 12 0, bytestream 1358 [in#0/rtsp @ 0x5623a501ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5623a501ee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T11:29:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 182.24, "last_started_at": "2026-07-02T11:29:21+09:00", "pid": 3286, "process": {"exists": false, "pid": 3286}, "restart_count": 8, "restart_count_per_hour": 1, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 180.527, "sample_count": 1918, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 73738, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:32:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.046, "last_frame_pushed_at": "2026-07-02T11:32:23+09:00", "last_frame_read_age_sec": 0.039, "last_frame_read_at": "2026-07-02T11:32:23+09:00", "mjpeg": {"age_sec": 0.032, "exists": true, "mtime": "2026-07-02T11:32:23.428861+09:00", "mtime_epoch": 1782959543.4288607, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190684}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:32:23+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 904.773, "name": "celery", "pid": 237, "rss_mb": 898.07, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 66, "pushed_frames": 66, "read_frames": 66, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.211, "pushed": 2.211, "read": 2.211}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.011, "count": 66, "max_ms": 8.198, "p95_ms": 7.358}, "gstreamer_push": {"avg_ms": 1.018, "count": 66, "max_ms": 2.953, "p95_ms": 1.844}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.893, "count": 66, "max_ms": 66.277, "p95_ms": 63.437}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.791, "count": 66, "max_ms": 95.99, "p95_ms": 89.649}, "yolo11_640_fire_11class": {"avg_ms": 48.537, "count": 66, "max_ms": 69.935, "p95_ms": 67.23}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.668, "count": 66, "max_ms": 89.963, "p95_ms": 87.362}}, "truncated_count": 0}, "processing_total": {"avg_ms": 163.413, "count": 66, "max_ms": 201.277, "p95_ms": 192.022}, "read_interval": {"avg_ms": 454.245, "count": 66, "max_ms": 18658.045, "p95_ms": 206.222}, "triton_wait": {"avg_ms": 63.091, "count": 66, "max_ms": 96.003, "p95_ms": 89.662}, "visualize": {"avg_ms": 7.649, "count": 66, "max_ms": 9.628, "p95_ms": 9.397}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.857, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 614}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4419.766, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T11:32:23+09:00", "last_error_summary": "[h264 @ 0x5623a5159080] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5623a5159080] error while decoding MB 0 29, bytestream 4610 [h264 @ 0x5623a5b71bc0] top block unavailable for requested intra mode -1 [h264 @ 0x5623a5b71bc0] error while decoding MB 12 0, bytestream 1358 [in#0/rtsp @ 0x5623a501ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5623a501ee00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 15, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:32:23+09:00", "last_error_summary": "[h264 @ 0x5623a5159080] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5623a5159080] error while decoding MB 0 29, bytestream 4610 [h264 @ 0x5623a5b71bc0] top block unavailable for requested intra mode -1 [h264 @ 0x5623a5b71bc0] error while decoding MB 12 0, bytestream 1358 [in#0/rtsp @ 0x5623a501ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5623a501ee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T11:32:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 182.369, "last_started_at": "2026-07-02T11:29:21+09:00", "pid": 3286, "process": {"exists": false, "pid": 3286}, "restart_count": 8, "restart_count_per_hour": 1, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 180.527, "sample_count": 1918, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 73739, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:32:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T11:32:23+09:00", "last_frame_read_age_sec": 0.167, "last_frame_read_at": "2026-07-02T11:32:23+09:00", "mjpeg": {"age_sec": 0.161, "exists": true, "mtime": "2026-07-02T11:32:23.428861+09:00", "mtime_epoch": 1782959543.4288607, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190684}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:32:23+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 904.773, "name": "celery", "pid": 237, "rss_mb": 898.07, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 67, "pushed_frames": 67, "read_frames": 66, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.234, "pushed": 2.234, "read": 2.201}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.005, "count": 66, "max_ms": 8.198, "p95_ms": 7.358}, "gstreamer_push": {"avg_ms": 1.017, "count": 67, "max_ms": 2.953, "p95_ms": 1.844}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.946, "count": 66, "max_ms": 66.277, "p95_ms": 63.437}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.843, "count": 66, "max_ms": 95.99, "p95_ms": 89.649}, "yolo11_640_fire_11class": {"avg_ms": 48.6, "count": 66, "max_ms": 69.935, "p95_ms": 67.23}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.729, "count": 66, "max_ms": 89.963, "p95_ms": 87.362}}, "truncated_count": 0}, "processing_total": {"avg_ms": 163.402, "count": 67, "max_ms": 201.277, "p95_ms": 192.022}, "read_interval": {"avg_ms": 454.245, "count": 66, "max_ms": 18658.045, "p95_ms": 206.222}, "triton_wait": {"avg_ms": 63.143, "count": 66, "max_ms": 96.003, "p95_ms": 89.662}, "visualize": {"avg_ms": 7.652, "count": 66, "max_ms": 9.628, "p95_ms": 9.397}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.985, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 614}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4419.883, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 15, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:32:23+09:00", "last_error_summary": "[h264 @ 0x5623a5159080] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5623a5159080] error while decoding MB 0 29, bytestream 4610 [h264 @ 0x5623a5b71bc0] top block unavailable for requested intra mode -1 [h264 @ 0x5623a5b71bc0] error while decoding MB 12 0, bytestream 1358 [in#0/rtsp @ 0x5623a501ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5623a501ee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T11:32:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 182.373, "last_started_at": "2026-07-02T11:29:21+09:00", "pid": 3286, "process": {"exists": false, "pid": 3286}, "restart_count": 8, "restart_count_per_hour": 1, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 180.527, "sample_count": 1918, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 73739, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:32:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T11:32:23+09:00", "last_frame_read_age_sec": 0.171, "last_frame_read_at": "2026-07-02T11:32:23+09:00", "mjpeg": {"age_sec": 0.164, "exists": true, "mtime": "2026-07-02T11:32:23.428861+09:00", "mtime_epoch": 1782959543.4288607, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190684}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:32:23+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 904.773, "name": "celery", "pid": 237, "rss_mb": 898.07, "state": "S (sleeping)", "threads": 120}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 67, "pushed_frames": 67, "read_frames": 66, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.234, "pushed": 2.234, "read": 2.201}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.005, "count": 66, "max_ms": 8.198, "p95_ms": 7.358}, "gstreamer_push": {"avg_ms": 1.017, "count": 67, "max_ms": 2.953, "p95_ms": 1.844}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.946, "count": 66, "max_ms": 66.277, "p95_ms": 63.437}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.843, "count": 66, "max_ms": 95.99, "p95_ms": 89.649}, "yolo11_640_fire_11class": {"avg_ms": 48.6, "count": 66, "max_ms": 69.935, "p95_ms": 67.23}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.729, "count": 66, "max_ms": 89.963, "p95_ms": 87.362}}, "truncated_count": 0}, "processing_total": {"avg_ms": 163.402, "count": 67, "max_ms": 201.277, "p95_ms": 192.022}, "read_interval": {"avg_ms": 454.245, "count": 66, "max_ms": 18658.045, "p95_ms": 206.222}, "triton_wait": {"avg_ms": 63.143, "count": 66, "max_ms": 96.003, "p95_ms": 89.662}, "visualize": {"avg_ms": 7.652, "count": 66, "max_ms": 9.628, "p95_ms": 9.397}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.989, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 0, "tracking_summary": 614}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4541.953, "limit_mb": 8796093022207.996}, "detail": {"pid": 3453, "restart_count": 9, "start_count": 10}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 6, "h264_decode_error_count": 15, "h264_decode_error_count_per_hour": 2, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:32:23+09:00", "last_error_summary": "[h264 @ 0x5623a5159080] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5623a5159080] error while decoding MB 0 29, bytestream 4610 [h264 @ 0x5623a5b71bc0] top block unavailable for requested intra mode -1 [h264 @ 0x5623a5b71bc0] error while decoding MB 12 0, bytestream 1358 [in#0/rtsp @ 0x5623a501ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5623a501ee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T11:32:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T11:32:26+09:00", "pid": 3453, "process": {"exists": true, "fd_count": 5, "hwm_mb": 149.719, "name": "ffmpeg", "pid": 3453, "rss_mb": 149.719, "state": "S (sleeping)", "threads": 106}, "restart_count": 9, "restart_count_per_hour": 2, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 10}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 180.636, "sample_count": 1881, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 73739, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:32:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.749, "last_frame_pushed_at": "2026-07-02T11:32:23+09:00", "last_frame_read_age_sec": 4.913, "last_frame_read_at": "2026-07-02T11:32:23+09:00", "mjpeg": {"age_sec": 4.734, "exists": true, "mtime": "2026-07-02T11:32:23.600860+09:00", "mtime_epoch": 1782959543.6008599, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190943}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:32:28+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 904.773, "name": "celery", "pid": 237, "rss_mb": 898.07, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 0, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 40, "pushed_frames": 40, "read_frames": 39, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.34, "pushed": 1.34, "read": 1.306}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.961, "count": 40, "max_ms": 7.572, "p95_ms": 6.988}, "gstreamer_push": {"avg_ms": 1.048, "count": 40, "max_ms": 2.953, "p95_ms": 1.931}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.619, "count": 40, "max_ms": 65.497, "p95_ms": 60.876}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.677, "count": 40, "max_ms": 89.946, "p95_ms": 86.319}, "yolo11_640_fire_11class": {"avg_ms": 46.571, "count": 40, "max_ms": 67.002, "p95_ms": 63.919}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.707, "count": 40, "max_ms": 89.963, "p95_ms": 85.553}}, "truncated_count": 0}, "processing_total": {"avg_ms": 159.813, "count": 40, "max_ms": 192.022, "p95_ms": 188.093}, "read_interval": {"avg_ms": 644.293, "count": 39, "max_ms": 18658.045, "p95_ms": 203.903}, "triton_wait": {"avg_ms": 60.296, "count": 40, "max_ms": 89.976, "p95_ms": 86.33}, "visualize": {"avg_ms": 7.508, "count": 40, "max_ms": 9.628, "p95_ms": 9.397}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.854, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 619}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4577.82, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55f903fa5e00] error while decoding MB 60 39, bytestream -11", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 16, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:34:31+09:00", "last_error_summary": "[h264 @ 0x55f903fa5e00] error while decoding MB 60 39, bytestream -11", "last_exit_at": "2026-07-02T11:32:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.929, "last_started_at": "2026-07-02T11:32:26+09:00", "pid": 3453, "process": {"exists": false, "pid": 3453}, "restart_count": 9, "restart_count_per_hour": 2, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 10}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 213.526, "sample_count": 1557, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 74343, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:34:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.057, "last_frame_pushed_at": "2026-07-02T11:34:31+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-02T11:34:31+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-02T11:34:31.224669+09:00", "mtime_epoch": 1782959671.2246695, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193919}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:34:31+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1165.184, "name": "celery", "pid": 237, "rss_mb": 1037.82, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 148, "pushed_frames": 148, "read_frames": 149, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.934, "pushed": 4.934, "read": 4.967}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.044, "count": 148, "max_ms": 10.791, "p95_ms": 8.371}, "gstreamer_push": {"avg_ms": 0.877, "count": 148, "max_ms": 1.263, "p95_ms": 0.929}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.555, "count": 148, "max_ms": 74.017, "p95_ms": 65.458}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.331, "count": 148, "max_ms": 107.696, "p95_ms": 94.417}, "yolo11_640_fire_11class": {"avg_ms": 51.402, "count": 148, "max_ms": 71.898, "p95_ms": 67.504}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.114, "count": 148, "max_ms": 94.511, "p95_ms": 89.1}}, "truncated_count": 0}, "processing_total": {"avg_ms": 192.469, "count": 148, "max_ms": 245.646, "p95_ms": 227.977}, "read_interval": {"avg_ms": 202.223, "count": 149, "max_ms": 255.5, "p95_ms": 236.707}, "triton_wait": {"avg_ms": 70.098, "count": 148, "max_ms": 107.71, "p95_ms": 94.429}, "visualize": {"avg_ms": 8.729, "count": 148, "max_ms": 11.323, "p95_ms": 10.153}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 619}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4577.082, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T11:34:31+09:00", "last_error_summary": "[h264 @ 0x55f903fa5e00] error while decoding MB 60 39, bytestream -11", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 16, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:34:31+09:00", "last_error_summary": "[h264 @ 0x55f903fa5e00] error while decoding MB 60 39, bytestream -11", "last_exit_at": "2026-07-02T11:34:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.1, "last_started_at": "2026-07-02T11:32:26+09:00", "pid": 3453, "process": {"exists": false, "pid": 3453}, "restart_count": 9, "restart_count_per_hour": 2, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 10}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 213.779, "sample_count": 1557, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 74344, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:34:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T11:34:31+09:00", "last_frame_read_age_sec": 0.22, "last_frame_read_at": "2026-07-02T11:34:31+09:00", "mjpeg": {"age_sec": 0.209, "exists": true, "mtime": "2026-07-02T11:34:31.224669+09:00", "mtime_epoch": 1782959671.2246695, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193919}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:34:31+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1165.184, "name": "celery", "pid": 237, "rss_mb": 1037.82, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 149, "pushed_frames": 149, "read_frames": 148, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.969, "pushed": 4.969, "read": 4.936}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.055, "count": 148, "max_ms": 10.791, "p95_ms": 8.371}, "gstreamer_push": {"avg_ms": 0.877, "count": 149, "max_ms": 1.263, "p95_ms": 0.929}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.546, "count": 148, "max_ms": 74.017, "p95_ms": 65.458}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.322, "count": 148, "max_ms": 107.696, "p95_ms": 94.417}, "yolo11_640_fire_11class": {"avg_ms": 51.393, "count": 148, "max_ms": 71.898, "p95_ms": 67.504}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.151, "count": 148, "max_ms": 94.511, "p95_ms": 89.1}}, "truncated_count": 0}, "processing_total": {"avg_ms": 192.623, "count": 149, "max_ms": 245.646, "p95_ms": 227.977}, "read_interval": {"avg_ms": 202.34, "count": 148, "max_ms": 255.5, "p95_ms": 236.707}, "triton_wait": {"avg_ms": 70.088, "count": 148, "max_ms": 107.71, "p95_ms": 94.429}, "visualize": {"avg_ms": 8.738, "count": 148, "max_ms": 11.323, "p95_ms": 10.153}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.984, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 619}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4577.148, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 16, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:34:31+09:00", "last_error_summary": "[h264 @ 0x55f903fa5e00] error while decoding MB 60 39, bytestream -11", "last_exit_at": "2026-07-02T11:34:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 125.103, "last_started_at": "2026-07-02T11:32:26+09:00", "pid": 3453, "process": {"exists": false, "pid": 3453}, "restart_count": 9, "restart_count_per_hour": 2, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 10}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 213.779, "sample_count": 1557, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 74344, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:34:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T11:34:31+09:00", "last_frame_read_age_sec": 0.224, "last_frame_read_at": "2026-07-02T11:34:31+09:00", "mjpeg": {"age_sec": 0.213, "exists": true, "mtime": "2026-07-02T11:34:31.224669+09:00", "mtime_epoch": 1782959671.2246695, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193919}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:34:31+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1165.184, "name": "celery", "pid": 237, "rss_mb": 1037.82, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 149, "pushed_frames": 149, "read_frames": 148, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.969, "pushed": 4.969, "read": 4.935}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.055, "count": 148, "max_ms": 10.791, "p95_ms": 8.371}, "gstreamer_push": {"avg_ms": 0.877, "count": 149, "max_ms": 1.263, "p95_ms": 0.929}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.546, "count": 148, "max_ms": 74.017, "p95_ms": 65.458}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.322, "count": 148, "max_ms": 107.696, "p95_ms": 94.417}, "yolo11_640_fire_11class": {"avg_ms": 51.393, "count": 148, "max_ms": 71.898, "p95_ms": 67.504}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.151, "count": 148, "max_ms": 94.511, "p95_ms": 89.1}}, "truncated_count": 0}, "processing_total": {"avg_ms": 192.623, "count": 149, "max_ms": 245.646, "p95_ms": 227.977}, "read_interval": {"avg_ms": 202.34, "count": 148, "max_ms": 255.5, "p95_ms": 236.707}, "triton_wait": {"avg_ms": 70.088, "count": 148, "max_ms": 107.71, "p95_ms": 94.429}, "visualize": {"avg_ms": 8.738, "count": 148, "max_ms": 11.323, "p95_ms": 10.153}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.988, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 619}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4680.25, "limit_mb": 8796093022207.996}, "detail": {"pid": 3621, "restart_count": 10, "start_count": 11}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 7, "h264_decode_error_count": 16, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:34:31+09:00", "last_error_summary": "[h264 @ 0x55f903fa5e00] error while decoding MB 60 39, bytestream -11", "last_exit_at": "2026-07-02T11:34:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T11:34:34+09:00", "pid": 3621, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.098, "name": "ffmpeg", "pid": 3621, "rss_mb": 139.098, "state": "S (sleeping)", "threads": 106}, "restart_count": 10, "restart_count_per_hour": 3, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 11}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 214.045, "sample_count": 1525, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 74344, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:34:31+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.989, "last_frame_pushed_at": "2026-07-02T11:34:31+09:00", "last_frame_read_age_sec": 5.205, "last_frame_read_at": "2026-07-02T11:34:31+09:00", "mjpeg": {"age_sec": 4.975, "exists": true, "mtime": "2026-07-02T11:34:31.444670+09:00", "mtime_epoch": 1782959671.4446697, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 193780}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:34:36+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1165.184, "name": "celery", "pid": 237, "rss_mb": 1037.82, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.109, "pushed": 4.109, "read": 4.076}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.086, "count": 122, "max_ms": 10.791, "p95_ms": 8.664}, "gstreamer_push": {"avg_ms": 0.876, "count": 123, "max_ms": 0.991, "p95_ms": 0.929}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.936, "count": 122, "max_ms": 74.017, "p95_ms": 65.458}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.958, "count": 122, "max_ms": 107.696, "p95_ms": 94.386}, "yolo11_640_fire_11class": {"avg_ms": 52.001, "count": 122, "max_ms": 71.898, "p95_ms": 67.65}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.353, "count": 122, "max_ms": 94.511, "p95_ms": 89.1}}, "truncated_count": 0}, "processing_total": {"avg_ms": 194.509, "count": 123, "max_ms": 245.646, "p95_ms": 232.711}, "read_interval": {"avg_ms": 204.214, "count": 122, "max_ms": 255.5, "p95_ms": 242.363}, "triton_wait": {"avg_ms": 71.06, "count": 122, "max_ms": 107.71, "p95_ms": 94.398}, "visualize": {"avg_ms": 8.732, "count": 122, "max_ms": 11.323, "p95_ms": 10.153}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.935, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 624}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4531.219, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5640a3981100] error while decoding MB 48 30, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 17, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:36:36+09:00", "last_error_summary": "[h264 @ 0x5640a3981100] error while decoding MB 48 30, bytestream -5", "last_exit_at": "2026-07-02T11:34:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 121.926, "last_started_at": "2026-07-02T11:34:34+09:00", "pid": 3621, "process": {"exists": false, "pid": 3621}, "restart_count": 10, "restart_count_per_hour": 3, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 11}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 221.221, "sample_count": 1376, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 74951, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:36:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-02T11:36:36+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-02T11:36:36+09:00", "mjpeg": {"age_sec": 0.051, "exists": true, "mtime": "2026-07-02T11:36:36.293042+09:00", "mtime_epoch": 1782959796.293042, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197733}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:36:36+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1165.184, "name": "celery", "pid": 237, "rss_mb": 1001.035, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 156, "pushed_frames": 156, "read_frames": 156, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.212, "pushed": 5.212, "read": 5.212}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.128, "count": 155, "max_ms": 10.397, "p95_ms": 8.442}, "gstreamer_push": {"avg_ms": 0.884, "count": 156, "max_ms": 1.028, "p95_ms": 0.963}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.701, "count": 155, "max_ms": 81.185, "p95_ms": 67.172}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.953, "count": 155, "max_ms": 102.301, "p95_ms": 91.809}, "yolo11_640_fire_11class": {"avg_ms": 50.145, "count": 155, "max_ms": 81.961, "p95_ms": 68.246}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.721, "count": 155, "max_ms": 102.412, "p95_ms": 92.461}}, "truncated_count": 0}, "processing_total": {"avg_ms": 183.442, "count": 156, "max_ms": 234.282, "p95_ms": 214.591}, "read_interval": {"avg_ms": 192.579, "count": 156, "max_ms": 245.024, "p95_ms": 223.195}, "triton_wait": {"avg_ms": 66.796, "count": 155, "max_ms": 102.424, "p95_ms": 93.572}, "visualize": {"avg_ms": 8.6, "count": 155, "max_ms": 11.337, "p95_ms": 9.868}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.928, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 624}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4536.75, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T11:36:36+09:00", "last_error_summary": "[h264 @ 0x5640a3981100] error while decoding MB 48 30, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 17, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:36:36+09:00", "last_error_summary": "[h264 @ 0x5640a3981100] error while decoding MB 48 30, bytestream -5", "last_exit_at": "2026-07-02T11:36:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 122.039, "last_started_at": "2026-07-02T11:34:34+09:00", "pid": 3621, "process": {"exists": false, "pid": 3621}, "restart_count": 10, "restart_count_per_hour": 3, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 11}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 221.221, "sample_count": 1377, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 74952, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:36:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T11:36:36+09:00", "last_frame_read_age_sec": 0.166, "last_frame_read_at": "2026-07-02T11:36:36+09:00", "mjpeg": {"age_sec": 0.164, "exists": true, "mtime": "2026-07-02T11:36:36.293042+09:00", "mtime_epoch": 1782959796.293042, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197733}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:36:36+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1165.184, "name": "celery", "pid": 237, "rss_mb": 1001.035, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 156, "pushed_frames": 156, "read_frames": 155, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.227, "pushed": 5.227, "read": 5.194}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.149, "count": 156, "max_ms": 10.4, "p95_ms": 8.472}, "gstreamer_push": {"avg_ms": 0.885, "count": 156, "max_ms": 1.028, "p95_ms": 0.963}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.649, "count": 156, "max_ms": 81.185, "p95_ms": 67.172}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.913, "count": 156, "max_ms": 102.301, "p95_ms": 91.809}, "yolo11_640_fire_11class": {"avg_ms": 50.11, "count": 156, "max_ms": 81.961, "p95_ms": 68.246}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.681, "count": 156, "max_ms": 102.412, "p95_ms": 92.461}}, "truncated_count": 0}, "processing_total": {"avg_ms": 183.399, "count": 156, "max_ms": 234.282, "p95_ms": 214.591}, "read_interval": {"avg_ms": 192.682, "count": 155, "max_ms": 245.024, "p95_ms": 223.195}, "triton_wait": {"avg_ms": 66.744, "count": 156, "max_ms": 102.424, "p95_ms": 93.572}, "visualize": {"avg_ms": 8.593, "count": 156, "max_ms": 11.337, "p95_ms": 9.868}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.843, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 624}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4532.84, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 17, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:36:36+09:00", "last_error_summary": "[h264 @ 0x5640a3981100] error while decoding MB 48 30, bytestream -5", "last_exit_at": "2026-07-02T11:36:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 122.043, "last_started_at": "2026-07-02T11:34:34+09:00", "pid": 3621, "process": {"exists": false, "pid": 3621}, "restart_count": 10, "restart_count_per_hour": 3, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 11}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 221.221, "sample_count": 1377, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 74952, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:36:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T11:36:36+09:00", "last_frame_read_age_sec": 0.17, "last_frame_read_at": "2026-07-02T11:36:36+09:00", "mjpeg": {"age_sec": 0.168, "exists": true, "mtime": "2026-07-02T11:36:36.293042+09:00", "mtime_epoch": 1782959796.293042, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197733}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:36:36+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1165.184, "name": "celery", "pid": 237, "rss_mb": 1001.035, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 156, "pushed_frames": 156, "read_frames": 155, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.227, "pushed": 5.227, "read": 5.193}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.149, "count": 156, "max_ms": 10.4, "p95_ms": 8.472}, "gstreamer_push": {"avg_ms": 0.885, "count": 156, "max_ms": 1.028, "p95_ms": 0.963}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.649, "count": 156, "max_ms": 81.185, "p95_ms": 67.172}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.913, "count": 156, "max_ms": 102.301, "p95_ms": 91.809}, "yolo11_640_fire_11class": {"avg_ms": 50.11, "count": 156, "max_ms": 81.961, "p95_ms": 68.246}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.681, "count": 156, "max_ms": 102.412, "p95_ms": 92.461}}, "truncated_count": 0}, "processing_total": {"avg_ms": 183.399, "count": 156, "max_ms": 234.282, "p95_ms": 214.591}, "read_interval": {"avg_ms": 192.682, "count": 155, "max_ms": 245.024, "p95_ms": 223.195}, "triton_wait": {"avg_ms": 66.744, "count": 156, "max_ms": 102.424, "p95_ms": 93.572}, "visualize": {"avg_ms": 8.593, "count": 156, "max_ms": 11.337, "p95_ms": 9.868}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.847, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 624}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4642.406, "limit_mb": 8796093022207.996}, "detail": {"pid": 3728, "restart_count": 11, "start_count": 12}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 8, "h264_decode_error_count": 17, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:36:36+09:00", "last_error_summary": "[h264 @ 0x5640a3981100] error while decoding MB 48 30, bytestream -5", "last_exit_at": "2026-07-02T11:36:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T11:36:38+09:00", "pid": 3728, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.535, "name": "ffmpeg", "pid": 3728, "rss_mb": 136.535, "state": "S (sleeping)", "threads": 106}, "restart_count": 11, "restart_count_per_hour": 4, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 12}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 221.523, "sample_count": 1353, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 74952, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:36:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.254, "last_frame_pushed_at": "2026-07-02T11:36:36+09:00", "last_frame_read_age_sec": 4.416, "last_frame_read_at": "2026-07-02T11:36:36+09:00", "mjpeg": {"age_sec": 4.231, "exists": true, "mtime": "2026-07-02T11:36:36.477043+09:00", "mtime_epoch": 1782959796.4770427, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197706}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:36:40+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1165.184, "name": "celery", "pid": 237, "rss_mb": 1001.035, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 134, "pushed_frames": 134, "read_frames": 134, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.467, "pushed": 4.467, "read": 4.467}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.226, "count": 134, "max_ms": 10.4, "p95_ms": 8.48}, "gstreamer_push": {"avg_ms": 0.881, "count": 134, "max_ms": 1.0, "p95_ms": 0.948}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.507, "count": 134, "max_ms": 81.185, "p95_ms": 67.172}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.596, "count": 134, "max_ms": 102.301, "p95_ms": 90.015}, "yolo11_640_fire_11class": {"avg_ms": 50.316, "count": 134, "max_ms": 81.961, "p95_ms": 68.246}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.608, "count": 134, "max_ms": 102.412, "p95_ms": 92.461}}, "truncated_count": 0}, "processing_total": {"avg_ms": 183.139, "count": 134, "max_ms": 234.282, "p95_ms": 214.591}, "read_interval": {"avg_ms": 192.461, "count": 134, "max_ms": 245.024, "p95_ms": 223.195}, "triton_wait": {"avg_ms": 66.622, "count": 134, "max_ms": 102.424, "p95_ms": 93.572}, "visualize": {"avg_ms": 8.569, "count": 134, "max_ms": 11.337, "p95_ms": 9.788}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 580}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4468.77, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5580a8b7f180] error while decoding MB 27 61, bytestream -45 [in#0/rtsp @ 0x5580a89c5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5580a89c5e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 9, "h264_decode_error_count": 9, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:38:25+09:00", "last_error_summary": "[h264 @ 0x5580a8b7f180] error while decoding MB 27 61, bytestream -45 [in#0/rtsp @ 0x5580a89c5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5580a89c5e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T11:27:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 643.932, "last_started_at": "2026-07-02T11:27:41+09:00", "pid": 3134, "process": {"exists": false, "pid": 3134}, "restart_count": 8, "restart_count_per_hour": 4, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.507, "sample_count": 2316, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 69647, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:38:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-02T11:38:25+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-02T11:38:25+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-02T11:38:25.853727+09:00", "mtime_epoch": 1782959905.8537269, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 244947}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T11:38:25+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1169.93, "name": "celery", "pid": 267, "rss_mb": 1034.383, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 237, "pushed_frames": 237, "read_frames": 237, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.919, "pushed": 7.919, "read": 7.919}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.672, "count": 237, "max_ms": 6.905, "p95_ms": 5.569}, "gstreamer_push": {"avg_ms": 1.081, "count": 237, "max_ms": 1.513, "p95_ms": 1.208}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.756, "count": 237, "max_ms": 74.918, "p95_ms": 61.846}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.381, "count": 237, "max_ms": 113.25, "p95_ms": 89.472}, "yolo11_640_fire_11class": {"avg_ms": 51.367, "count": 237, "max_ms": 79.666, "p95_ms": 65.25}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.932, "count": 237, "max_ms": 96.439, "p95_ms": 84.907}}, "truncated_count": 0}, "processing_total": {"avg_ms": 114.033, "count": 237, "max_ms": 168.727, "p95_ms": 144.883}, "read_interval": {"avg_ms": 126.638, "count": 237, "max_ms": 184.409, "p95_ms": 159.204}, "triton_wait": {"avg_ms": 62.01, "count": 237, "max_ms": 113.262, "p95_ms": 91.831}, "visualize": {"avg_ms": 5.061, "count": 237, "max_ms": 7.823, "p95_ms": 5.824}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.928, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 580}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4474.449, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T11:38:25+09:00", "last_error_summary": "[h264 @ 0x5580a8b7f180] error while decoding MB 27 61, bytestream -45 [in#0/rtsp @ 0x5580a89c5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5580a89c5e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 9, "h264_decode_error_count": 9, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:38:25+09:00", "last_error_summary": "[h264 @ 0x5580a8b7f180] error while decoding MB 27 61, bytestream -45 [in#0/rtsp @ 0x5580a89c5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5580a89c5e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T11:38:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 644.005, "last_started_at": "2026-07-02T11:27:41+09:00", "pid": 3134, "process": {"exists": false, "pid": 3134}, "restart_count": 8, "restart_count_per_hour": 4, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.507, "sample_count": 2316, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 69648, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:38:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T11:38:25+09:00", "last_frame_read_age_sec": 0.126, "last_frame_read_at": "2026-07-02T11:38:25+09:00", "mjpeg": {"age_sec": 0.117, "exists": true, "mtime": "2026-07-02T11:38:25.853727+09:00", "mtime_epoch": 1782959905.8537269, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 244947}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T11:38:25+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1169.93, "name": "celery", "pid": 267, "rss_mb": 1034.383, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 237, "pushed_frames": 238, "read_frames": 237, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.9, "pushed": 7.933, "read": 7.9}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.669, "count": 237, "max_ms": 6.905, "p95_ms": 5.569}, "gstreamer_push": {"avg_ms": 1.08, "count": 238, "max_ms": 1.513, "p95_ms": 1.208}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.724, "count": 237, "max_ms": 74.918, "p95_ms": 61.846}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.315, "count": 237, "max_ms": 113.25, "p95_ms": 89.472}, "yolo11_640_fire_11class": {"avg_ms": 51.337, "count": 237, "max_ms": 79.666, "p95_ms": 65.25}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.841, "count": 237, "max_ms": 96.439, "p95_ms": 84.907}}, "truncated_count": 0}, "processing_total": {"avg_ms": 113.95, "count": 237, "max_ms": 168.727, "p95_ms": 144.883}, "read_interval": {"avg_ms": 126.638, "count": 237, "max_ms": 184.409, "p95_ms": 159.204}, "triton_wait": {"avg_ms": 61.939, "count": 237, "max_ms": 113.262, "p95_ms": 91.831}, "visualize": {"avg_ms": 5.058, "count": 237, "max_ms": 7.823, "p95_ms": 5.793}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 580}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4471.023, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 9, "h264_decode_error_count": 9, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:38:25+09:00", "last_error_summary": "[h264 @ 0x5580a8b7f180] error while decoding MB 27 61, bytestream -45 [in#0/rtsp @ 0x5580a89c5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5580a89c5e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T11:38:25+09:00", "last_returncode": 0, "last_stable_duration_sec": 644.011, "last_started_at": "2026-07-02T11:27:41+09:00", "pid": 3134, "process": {"exists": false, "pid": 3134}, "restart_count": 8, "restart_count_per_hour": 4, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 9}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.507, "sample_count": 2316, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 69648, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:38:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-02T11:38:25+09:00", "last_frame_read_age_sec": 0.132, "last_frame_read_at": "2026-07-02T11:38:25+09:00", "mjpeg": {"age_sec": 0.123, "exists": true, "mtime": "2026-07-02T11:38:25.853727+09:00", "mtime_epoch": 1782959905.8537269, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 244947}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T11:38:25+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1169.93, "name": "celery", "pid": 267, "rss_mb": 1034.383, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 237, "pushed_frames": 237, "read_frames": 237, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.901, "pushed": 7.901, "read": 7.901}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.669, "count": 237, "max_ms": 6.905, "p95_ms": 5.569}, "gstreamer_push": {"avg_ms": 1.081, "count": 237, "max_ms": 1.513, "p95_ms": 1.208}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.724, "count": 237, "max_ms": 74.918, "p95_ms": 61.846}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.315, "count": 237, "max_ms": 113.25, "p95_ms": 89.472}, "yolo11_640_fire_11class": {"avg_ms": 51.337, "count": 237, "max_ms": 79.666, "p95_ms": 65.25}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.841, "count": 237, "max_ms": 96.439, "p95_ms": 84.907}}, "truncated_count": 0}, "processing_total": {"avg_ms": 113.95, "count": 237, "max_ms": 168.727, "p95_ms": 144.883}, "read_interval": {"avg_ms": 126.638, "count": 237, "max_ms": 184.409, "p95_ms": 159.204}, "triton_wait": {"avg_ms": 61.939, "count": 237, "max_ms": 113.262, "p95_ms": 91.831}, "visualize": {"avg_ms": 5.058, "count": 237, "max_ms": 7.823, "p95_ms": 5.793}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 580}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4508.297, "limit_mb": 8796093022207.996}, "detail": {"pid": 3835, "restart_count": 9, "start_count": 10}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 9, "h264_decode_error_count": 9, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:38:25+09:00", "last_error_summary": "[h264 @ 0x5580a8b7f180] error while decoding MB 27 61, bytestream -45 [in#0/rtsp @ 0x5580a89c5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5580a89c5e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T11:38:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T11:38:28+09:00", "pid": 3835, "process": {"exists": true, "fd_count": 5, "hwm_mb": 65.242, "name": "ffmpeg", "pid": 3835, "rss_mb": 65.242, "state": "S (sleeping)", "threads": 19}, "restart_count": 9, "restart_count_per_hour": 5, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 10}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.49, "sample_count": 2285, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 69648, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:38:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 4.1, "last_frame_pushed_at": "2026-07-02T11:38:25+09:00", "last_frame_read_age_sec": 4.221, "last_frame_read_at": "2026-07-02T11:38:25+09:00", "mjpeg": {"age_sec": 4.079, "exists": true, "mtime": "2026-07-02T11:38:25.985728+09:00", "mtime_epoch": 1782959905.9857278, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 244938}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T11:38:30+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1169.93, "name": "celery", "pid": 267, "rss_mb": 1034.383, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 205, "pushed_frames": 205, "read_frames": 204, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.843, "pushed": 6.843, "read": 6.81}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 4.688, "count": 204, "max_ms": 6.905, "p95_ms": 5.532}, "gstreamer_push": {"avg_ms": 1.08, "count": 205, "max_ms": 1.358, "p95_ms": 1.204}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.72, "count": 204, "max_ms": 74.918, "p95_ms": 62.022}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.263, "count": 204, "max_ms": 113.25, "p95_ms": 90.273}, "yolo11_640_fire_11class": {"avg_ms": 51.281, "count": 204, "max_ms": 79.666, "p95_ms": 65.609}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.945, "count": 204, "max_ms": 96.439, "p95_ms": 87.889}}, "truncated_count": 0}, "processing_total": {"avg_ms": 114.155, "count": 205, "max_ms": 168.727, "p95_ms": 144.883}, "read_interval": {"avg_ms": 126.715, "count": 204, "max_ms": 184.409, "p95_ms": 159.204}, "triton_wait": {"avg_ms": 62.033, "count": 204, "max_ms": 113.262, "p95_ms": 92.063}, "visualize": {"avg_ms": 5.094, "count": 204, "max_ms": 7.823, "p95_ms": 5.744}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.958, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 629}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4528.609, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5629252707c0] error while decoding MB 37 29, bytestream -34", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 9, "h264_decode_error_count": 18, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:38:42+09:00", "last_error_summary": "[h264 @ 0x5629252707c0] error while decoding MB 37 29, bytestream -34", "last_exit_at": "2026-07-02T11:36:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.415, "last_started_at": "2026-07-02T11:36:38+09:00", "pid": 3728, "process": {"exists": false, "pid": 3728}, "restart_count": 11, "restart_count_per_hour": 4, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 12}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 217.126, "sample_count": 1479, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 75577, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:38:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.05, "last_frame_pushed_at": "2026-07-02T11:38:42+09:00", "last_frame_read_age_sec": 0.041, "last_frame_read_at": "2026-07-02T11:38:42+09:00", "mjpeg": {"age_sec": 0.035, "exists": true, "mtime": "2026-07-02T11:38:42.085852+09:00", "mtime_epoch": 1782959922.0858524, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198559}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:38:42+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1165.184, "name": "celery", "pid": 237, "rss_mb": 1001.293, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 155, "pushed_frames": 155, "read_frames": 155, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.203, "pushed": 5.203, "read": 5.203}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.187, "count": 155, "max_ms": 10.532, "p95_ms": 8.652}, "gstreamer_push": {"avg_ms": 0.88, "count": 155, "max_ms": 1.043, "p95_ms": 0.964}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.454, "count": 155, "max_ms": 74.207, "p95_ms": 62.946}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.041, "count": 155, "max_ms": 107.72, "p95_ms": 92.396}, "yolo11_640_fire_11class": {"avg_ms": 49.179, "count": 155, "max_ms": 78.432, "p95_ms": 68.507}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.955, "count": 155, "max_ms": 101.516, "p95_ms": 89.553}}, "truncated_count": 0}, "processing_total": {"avg_ms": 183.564, "count": 155, "max_ms": 229.005, "p95_ms": 216.136}, "read_interval": {"avg_ms": 193.288, "count": 155, "max_ms": 238.479, "p95_ms": 226.668}, "triton_wait": {"avg_ms": 66.956, "count": 155, "max_ms": 107.733, "p95_ms": 93.539}, "visualize": {"avg_ms": 8.457, "count": 155, "max_ms": 11.599, "p95_ms": 10.05}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.788, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 629}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4534.363, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T11:38:42+09:00", "last_error_summary": "[h264 @ 0x5629252707c0] error while decoding MB 37 29, bytestream -34", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 9, "h264_decode_error_count": 18, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:38:42+09:00", "last_error_summary": "[h264 @ 0x5629252707c0] error while decoding MB 37 29, bytestream -34", "last_exit_at": "2026-07-02T11:38:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.573, "last_started_at": "2026-07-02T11:36:38+09:00", "pid": 3728, "process": {"exists": false, "pid": 3728}, "restart_count": 11, "restart_count_per_hour": 4, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 12}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 217.126, "sample_count": 1479, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 75578, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:38:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T11:38:42+09:00", "last_frame_read_age_sec": 0.2, "last_frame_read_at": "2026-07-02T11:38:42+09:00", "mjpeg": {"age_sec": 0.194, "exists": true, "mtime": "2026-07-02T11:38:42.085852+09:00", "mtime_epoch": 1782959922.0858524, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198559}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:38:42+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1165.184, "name": "celery", "pid": 237, "rss_mb": 1001.293, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 156, "pushed_frames": 156, "read_frames": 155, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.209, "pushed": 5.209, "read": 5.176}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.185, "count": 155, "max_ms": 10.532, "p95_ms": 8.652}, "gstreamer_push": {"avg_ms": 0.88, "count": 156, "max_ms": 1.043, "p95_ms": 0.964}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.367, "count": 155, "max_ms": 74.207, "p95_ms": 62.946}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.966, "count": 155, "max_ms": 107.72, "p95_ms": 92.396}, "yolo11_640_fire_11class": {"avg_ms": 49.035, "count": 155, "max_ms": 78.432, "p95_ms": 66.558}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.747, "count": 155, "max_ms": 101.516, "p95_ms": 89.553}}, "truncated_count": 0}, "processing_total": {"avg_ms": 183.639, "count": 156, "max_ms": 229.005, "p95_ms": 216.136}, "read_interval": {"avg_ms": 193.288, "count": 155, "max_ms": 238.479, "p95_ms": 226.668}, "triton_wait": {"avg_ms": 66.87, "count": 155, "max_ms": 107.733, "p95_ms": 93.539}, "visualize": {"avg_ms": 8.455, "count": 155, "max_ms": 11.599, "p95_ms": 10.05}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.947, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 629}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4534.355, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 9, "h264_decode_error_count": 18, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:38:42+09:00", "last_error_summary": "[h264 @ 0x5629252707c0] error while decoding MB 37 29, bytestream -34", "last_exit_at": "2026-07-02T11:38:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 123.577, "last_started_at": "2026-07-02T11:36:38+09:00", "pid": 3728, "process": {"exists": false, "pid": 3728}, "restart_count": 11, "restart_count_per_hour": 4, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 12}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 217.126, "sample_count": 1479, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 75578, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:38:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T11:38:42+09:00", "last_frame_read_age_sec": 0.203, "last_frame_read_at": "2026-07-02T11:38:42+09:00", "mjpeg": {"age_sec": 0.198, "exists": true, "mtime": "2026-07-02T11:38:42.085852+09:00", "mtime_epoch": 1782959922.0858524, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198559}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:38:42+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1165.184, "name": "celery", "pid": 237, "rss_mb": 1001.293, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 156, "pushed_frames": 156, "read_frames": 155, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.209, "pushed": 5.209, "read": 5.175}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.185, "count": 155, "max_ms": 10.532, "p95_ms": 8.652}, "gstreamer_push": {"avg_ms": 0.88, "count": 156, "max_ms": 1.043, "p95_ms": 0.964}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.367, "count": 155, "max_ms": 74.207, "p95_ms": 62.946}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.966, "count": 155, "max_ms": 107.72, "p95_ms": 92.396}, "yolo11_640_fire_11class": {"avg_ms": 49.035, "count": 155, "max_ms": 78.432, "p95_ms": 66.558}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.747, "count": 155, "max_ms": 101.516, "p95_ms": 89.553}}, "truncated_count": 0}, "processing_total": {"avg_ms": 183.639, "count": 156, "max_ms": 229.005, "p95_ms": 216.136}, "read_interval": {"avg_ms": 193.288, "count": 155, "max_ms": 238.479, "p95_ms": 226.668}, "triton_wait": {"avg_ms": 66.87, "count": 155, "max_ms": 107.733, "p95_ms": 93.539}, "visualize": {"avg_ms": 8.455, "count": 155, "max_ms": 11.599, "p95_ms": 10.05}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.95, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 629}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4652.777, "limit_mb": 8796093022207.996}, "detail": {"pid": 3987, "restart_count": 12, "start_count": 13}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 9, "h264_decode_error_count": 18, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:38:42+09:00", "last_error_summary": "[h264 @ 0x5629252707c0] error while decoding MB 37 29, bytestream -34", "last_exit_at": "2026-07-02T11:38:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T11:38:45+09:00", "pid": 3987, "process": {"exists": true, "fd_count": 5, "hwm_mb": 146.879, "name": "ffmpeg", "pid": 3987, "rss_mb": 146.879, "state": "S (sleeping)", "threads": 106}, "restart_count": 12, "restart_count_per_hour": 5, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 13}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 217.126, "sample_count": 1454, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 75578, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:38:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.954, "last_frame_pushed_at": "2026-07-02T11:38:42+09:00", "last_frame_read_age_sec": 5.15, "last_frame_read_at": "2026-07-02T11:38:42+09:00", "mjpeg": {"age_sec": 4.945, "exists": true, "mtime": "2026-07-02T11:38:42.285854+09:00", "mtime_epoch": 1782959922.285854, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198446}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:38:47+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1165.184, "name": "celery", "pid": 237, "rss_mb": 1001.293, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 131, "pushed_frames": 131, "read_frames": 130, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.375, "pushed": 4.375, "read": 4.341}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.224, "count": 130, "max_ms": 9.37, "p95_ms": 8.571}, "gstreamer_push": {"avg_ms": 0.882, "count": 131, "max_ms": 1.043, "p95_ms": 0.964}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.078, "count": 130, "max_ms": 74.207, "p95_ms": 62.621}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.279, "count": 130, "max_ms": 107.72, "p95_ms": 91.661}, "yolo11_640_fire_11class": {"avg_ms": 48.506, "count": 130, "max_ms": 76.169, "p95_ms": 65.963}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.212, "count": 130, "max_ms": 101.516, "p95_ms": 89.032}}, "truncated_count": 0}, "processing_total": {"avg_ms": 182.47, "count": 131, "max_ms": 225.708, "p95_ms": 215.899}, "read_interval": {"avg_ms": 192.129, "count": 130, "max_ms": 235.442, "p95_ms": 226.668}, "triton_wait": {"avg_ms": 66.316, "count": 130, "max_ms": 107.733, "p95_ms": 93.539}, "visualize": {"avg_ms": 8.507, "count": 130, "max_ms": 11.216, "p95_ms": 10.05}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.945, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 584}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4482.617, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55bc91004800] error while decoding MB 43 49, bytestream -17 [in#0/rtsp @ 0x55bc90e5ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55bc90e5ce00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 10, "h264_decode_error_count": 10, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:40:08+09:00", "last_error_summary": "[h264 @ 0x55bc91004800] error while decoding MB 43 49, bytestream -17 [in#0/rtsp @ 0x55bc90e5ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55bc90e5ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T11:38:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 100.038, "last_started_at": "2026-07-02T11:38:28+09:00", "pid": 3835, "process": {"exists": false, "pid": 3835}, "restart_count": 9, "restart_count_per_hour": 5, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 10}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.471, "sample_count": 2041, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 70186, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:40:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.075, "last_frame_pushed_at": "2026-07-02T11:40:08+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-02T11:40:08+09:00", "mjpeg": {"age_sec": 0.056, "exists": true, "mtime": "2026-07-02T11:40:08.046608+09:00", "mtime_epoch": 1782960008.0466082, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 244946}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T11:40:08+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1169.93, "name": "celery", "pid": 267, "rss_mb": 1034.641, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.687, "pushed": 0.687, "read": 0.744}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.856, "count": 12, "max_ms": 5.906, "p95_ms": 5.906}, "gstreamer_push": {"avg_ms": 1.039, "count": 12, "max_ms": 1.245, "p95_ms": 1.245}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.718, "count": 12, "max_ms": 65.711, "p95_ms": 65.711}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.647, "count": 12, "max_ms": 92.122, "p95_ms": 92.122}, "yolo11_640_fire_11class": {"avg_ms": 52.859, "count": 12, "max_ms": 64.831, "p95_ms": 64.831}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.17, "count": 12, "max_ms": 89.608, "p95_ms": 89.608}}, "truncated_count": 0}, "processing_total": {"avg_ms": 112.401, "count": 12, "max_ms": 133.56, "p95_ms": 133.56}, "read_interval": {"avg_ms": 2469.601, "count": 13, "max_ms": 16046.602, "p95_ms": 16046.602}, "triton_wait": {"avg_ms": 63.146, "count": 12, "max_ms": 92.135, "p95_ms": 92.135}, "visualize": {"avg_ms": 5.142, "count": 12, "max_ms": 5.843, "p95_ms": 5.843}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.479, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 584}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4482.074, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T11:40:08+09:00", "last_error_summary": "[h264 @ 0x55bc91004800] error while decoding MB 43 49, bytestream -17 [in#0/rtsp @ 0x55bc90e5ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55bc90e5ce00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 10, "h264_decode_error_count": 10, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:40:08+09:00", "last_error_summary": "[h264 @ 0x55bc91004800] error while decoding MB 43 49, bytestream -17 [in#0/rtsp @ 0x55bc90e5ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55bc90e5ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T11:40:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 100.103, "last_started_at": "2026-07-02T11:38:28+09:00", "pid": 3835, "process": {"exists": false, "pid": 3835}, "restart_count": 9, "restart_count_per_hour": 5, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 10}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.471, "sample_count": 2042, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 70187, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:40:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T11:40:08+09:00", "last_frame_read_age_sec": 0.13, "last_frame_read_at": "2026-07-02T11:40:08+09:00", "mjpeg": {"age_sec": 0.121, "exists": true, "mtime": "2026-07-02T11:40:08.046608+09:00", "mtime_epoch": 1782960008.0466082, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 244946}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T11:40:08+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1169.93, "name": "celery", "pid": 267, "rss_mb": 1034.641, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.741, "pushed": 0.741, "read": 0.741}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.9, "count": 13, "max_ms": 5.906, "p95_ms": 5.906}, "gstreamer_push": {"avg_ms": 1.057, "count": 13, "max_ms": 1.273, "p95_ms": 1.273}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.424, "count": 13, "max_ms": 65.711, "p95_ms": 65.711}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.239, "count": 13, "max_ms": 92.122, "p95_ms": 92.122}, "yolo11_640_fire_11class": {"avg_ms": 52.713, "count": 13, "max_ms": 64.831, "p95_ms": 64.831}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.704, "count": 13, "max_ms": 89.608, "p95_ms": 89.608}}, "truncated_count": 0}, "processing_total": {"avg_ms": 113.177, "count": 13, "max_ms": 133.56, "p95_ms": 133.56}, "read_interval": {"avg_ms": 2469.601, "count": 13, "max_ms": 16046.602, "p95_ms": 16046.602}, "triton_wait": {"avg_ms": 62.624, "count": 13, "max_ms": 92.135, "p95_ms": 92.135}, "visualize": {"avg_ms": 5.393, "count": 13, "max_ms": 8.405, "p95_ms": 8.405}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.543, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 584}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4481.996, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 10, "h264_decode_error_count": 10, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:40:08+09:00", "last_error_summary": "[h264 @ 0x55bc91004800] error while decoding MB 43 49, bytestream -17 [in#0/rtsp @ 0x55bc90e5ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55bc90e5ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T11:40:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 100.11, "last_started_at": "2026-07-02T11:38:28+09:00", "pid": 3835, "process": {"exists": false, "pid": 3835}, "restart_count": 9, "restart_count_per_hour": 5, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 10}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.471, "sample_count": 2042, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 70187, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:40:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-02T11:40:08+09:00", "last_frame_read_age_sec": 0.137, "last_frame_read_at": "2026-07-02T11:40:08+09:00", "mjpeg": {"age_sec": 0.128, "exists": true, "mtime": "2026-07-02T11:40:08.046608+09:00", "mtime_epoch": 1782960008.0466082, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 244946}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T11:40:08+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1169.93, "name": "celery", "pid": 267, "rss_mb": 1034.641, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.741, "pushed": 0.741, "read": 0.741}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.9, "count": 13, "max_ms": 5.906, "p95_ms": 5.906}, "gstreamer_push": {"avg_ms": 1.057, "count": 13, "max_ms": 1.273, "p95_ms": 1.273}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.424, "count": 13, "max_ms": 65.711, "p95_ms": 65.711}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.239, "count": 13, "max_ms": 92.122, "p95_ms": 92.122}, "yolo11_640_fire_11class": {"avg_ms": 52.713, "count": 13, "max_ms": 64.831, "p95_ms": 64.831}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.704, "count": 13, "max_ms": 89.608, "p95_ms": 89.608}}, "truncated_count": 0}, "processing_total": {"avg_ms": 113.177, "count": 13, "max_ms": 133.56, "p95_ms": 133.56}, "read_interval": {"avg_ms": 2469.601, "count": 13, "max_ms": 16046.602, "p95_ms": 16046.602}, "triton_wait": {"avg_ms": 62.624, "count": 13, "max_ms": 92.135, "p95_ms": 92.135}, "visualize": {"avg_ms": 5.393, "count": 13, "max_ms": 8.405, "p95_ms": 8.405}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.55, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 584}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4645.703, "limit_mb": 8796093022207.996}, "detail": {"pid": 4094, "restart_count": 10, "start_count": 11}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 10, "h264_decode_error_count": 10, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:40:08+09:00", "last_error_summary": "[h264 @ 0x55bc91004800] error while decoding MB 43 49, bytestream -17 [in#0/rtsp @ 0x55bc90e5ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55bc90e5ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T11:40:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T11:40:10+09:00", "pid": 4094, "process": {"exists": true, "fd_count": 5, "hwm_mb": 197.695, "name": "ffmpeg", "pid": 4094, "rss_mb": 197.695, "state": "S (sleeping)", "threads": 106}, "restart_count": 10, "restart_count_per_hour": 6, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 11}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.507, "sample_count": 2008, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 70187, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:40:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 4.249, "last_frame_pushed_at": "2026-07-02T11:40:08+09:00", "last_frame_read_age_sec": 4.373, "last_frame_read_at": "2026-07-02T11:40:08+09:00", "mjpeg": {"age_sec": 4.232, "exists": true, "mtime": "2026-07-02T11:40:08.178609+09:00", "mtime_epoch": 1782960008.1786094, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 244914}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T11:40:12+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1169.93, "name": "celery", "pid": 267, "rss_mb": 1034.641, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.597, "pushed": 0.597, "read": 0.597}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.9, "count": 13, "max_ms": 5.906, "p95_ms": 5.906}, "gstreamer_push": {"avg_ms": 1.057, "count": 13, "max_ms": 1.273, "p95_ms": 1.273}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.424, "count": 13, "max_ms": 65.711, "p95_ms": 65.711}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.239, "count": 13, "max_ms": 92.122, "p95_ms": 92.122}, "yolo11_640_fire_11class": {"avg_ms": 52.713, "count": 13, "max_ms": 64.831, "p95_ms": 64.831}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.704, "count": 13, "max_ms": 89.608, "p95_ms": 89.608}}, "truncated_count": 0}, "processing_total": {"avg_ms": 113.177, "count": 13, "max_ms": 133.56, "p95_ms": 133.56}, "read_interval": {"avg_ms": 2469.601, "count": 13, "max_ms": 16046.602, "p95_ms": 16046.602}, "triton_wait": {"avg_ms": 62.624, "count": 13, "max_ms": 92.135, "p95_ms": 92.135}, "visualize": {"avg_ms": 5.393, "count": 13, "max_ms": 8.405, "p95_ms": 8.405}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.786, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 634}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4551.793, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5561f371cac0] error while decoding MB 43 15, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 10, "h264_decode_error_count": 19, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:40:49+09:00", "last_error_summary": "[h264 @ 0x5561f371cac0] error while decoding MB 43 15, bytestream -5", "last_exit_at": "2026-07-02T11:38:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.64, "last_started_at": "2026-07-02T11:38:45+09:00", "pid": 3987, "process": {"exists": false, "pid": 3987}, "restart_count": 12, "restart_count_per_hour": 5, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 13}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 218.662, "sample_count": 1476, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 76187, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:40:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.061, "last_frame_pushed_at": "2026-07-02T11:40:49+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-02T11:40:49+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-02T11:40:49.827025+09:00", "mtime_epoch": 1782960049.8270254, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204094}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:40:49+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1165.184, "name": "celery", "pid": 237, "rss_mb": 1001.551, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 138, "pushed_frames": 138, "read_frames": 138, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.611, "pushed": 4.611, "read": 4.611}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.317, "count": 137, "max_ms": 12.322, "p95_ms": 9.778}, "gstreamer_push": {"avg_ms": 0.864, "count": 138, "max_ms": 1.229, "p95_ms": 0.938}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.524, "count": 137, "max_ms": 75.292, "p95_ms": 66.387}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.057, "count": 137, "max_ms": 99.046, "p95_ms": 92.789}, "yolo11_640_fire_11class": {"avg_ms": 50.816, "count": 137, "max_ms": 86.741, "p95_ms": 66.568}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.262, "count": 137, "max_ms": 104.676, "p95_ms": 93.482}}, "truncated_count": 0}, "processing_total": {"avg_ms": 208.249, "count": 138, "max_ms": 291.247, "p95_ms": 239.412}, "read_interval": {"avg_ms": 217.86, "count": 138, "max_ms": 301.774, "p95_ms": 249.376}, "triton_wait": {"avg_ms": 69.563, "count": 137, "max_ms": 104.689, "p95_ms": 95.049}, "visualize": {"avg_ms": 9.597, "count": 137, "max_ms": 12.81, "p95_ms": 10.96}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.93, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 634}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4550.875, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T11:40:49+09:00", "last_error_summary": "[h264 @ 0x5561f371cac0] error while decoding MB 43 15, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 10, "h264_decode_error_count": 19, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:40:49+09:00", "last_error_summary": "[h264 @ 0x5561f371cac0] error while decoding MB 43 15, bytestream -5", "last_exit_at": "2026-07-02T11:40:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.83, "last_started_at": "2026-07-02T11:38:45+09:00", "pid": 3987, "process": {"exists": false, "pid": 3987}, "restart_count": 12, "restart_count_per_hour": 5, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 13}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 218.662, "sample_count": 1476, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 76188, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:40:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T11:40:50+09:00", "last_frame_read_age_sec": 0.243, "last_frame_read_at": "2026-07-02T11:40:49+09:00", "mjpeg": {"age_sec": 0.233, "exists": true, "mtime": "2026-07-02T11:40:49.827025+09:00", "mtime_epoch": 1782960049.8270254, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204094}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:40:50+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1165.184, "name": "celery", "pid": 237, "rss_mb": 1001.551, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 138, "pushed_frames": 138, "read_frames": 137, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.612, "pushed": 4.612, "read": 4.578}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.325, "count": 137, "max_ms": 12.322, "p95_ms": 9.778}, "gstreamer_push": {"avg_ms": 0.864, "count": 138, "max_ms": 1.229, "p95_ms": 0.938}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.575, "count": 137, "max_ms": 75.292, "p95_ms": 66.387}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.218, "count": 137, "max_ms": 99.046, "p95_ms": 92.789}, "yolo11_640_fire_11class": {"avg_ms": 50.807, "count": 137, "max_ms": 86.741, "p95_ms": 66.568}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.309, "count": 137, "max_ms": 104.676, "p95_ms": 93.482}}, "truncated_count": 0}, "processing_total": {"avg_ms": 208.6, "count": 138, "max_ms": 291.247, "p95_ms": 239.412}, "read_interval": {"avg_ms": 218.017, "count": 137, "max_ms": 301.774, "p95_ms": 249.376}, "triton_wait": {"avg_ms": 69.725, "count": 137, "max_ms": 104.689, "p95_ms": 95.049}, "visualize": {"avg_ms": 9.606, "count": 137, "max_ms": 12.81, "p95_ms": 10.96}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.923, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 634}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4550.852, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 10, "h264_decode_error_count": 19, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:40:49+09:00", "last_error_summary": "[h264 @ 0x5561f371cac0] error while decoding MB 43 15, bytestream -5", "last_exit_at": "2026-07-02T11:40:50+09:00", "last_returncode": 0, "last_stable_duration_sec": 124.834, "last_started_at": "2026-07-02T11:38:45+09:00", "pid": 3987, "process": {"exists": false, "pid": 3987}, "restart_count": 12, "restart_count_per_hour": 5, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 13}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 218.662, "sample_count": 1476, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 76188, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:40:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-02T11:40:50+09:00", "last_frame_read_age_sec": 0.246, "last_frame_read_at": "2026-07-02T11:40:49+09:00", "mjpeg": {"age_sec": 0.236, "exists": true, "mtime": "2026-07-02T11:40:49.827025+09:00", "mtime_epoch": 1782960049.8270254, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204094}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:40:50+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1165.184, "name": "celery", "pid": 237, "rss_mb": 1001.551, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 138, "pushed_frames": 138, "read_frames": 137, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.611, "pushed": 4.611, "read": 4.578}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.325, "count": 137, "max_ms": 12.322, "p95_ms": 9.778}, "gstreamer_push": {"avg_ms": 0.864, "count": 138, "max_ms": 1.229, "p95_ms": 0.938}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.575, "count": 137, "max_ms": 75.292, "p95_ms": 66.387}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.218, "count": 137, "max_ms": 99.046, "p95_ms": 92.789}, "yolo11_640_fire_11class": {"avg_ms": 50.807, "count": 137, "max_ms": 86.741, "p95_ms": 66.568}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.309, "count": 137, "max_ms": 104.676, "p95_ms": 93.482}}, "truncated_count": 0}, "processing_total": {"avg_ms": 208.6, "count": 138, "max_ms": 291.247, "p95_ms": 239.412}, "read_interval": {"avg_ms": 218.017, "count": 137, "max_ms": 301.774, "p95_ms": 249.376}, "triton_wait": {"avg_ms": 69.725, "count": 137, "max_ms": 104.689, "p95_ms": 95.049}, "visualize": {"avg_ms": 9.606, "count": 137, "max_ms": 12.81, "p95_ms": 10.96}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.926, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 634}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4652.988, "limit_mb": 8796093022207.996}, "detail": {"pid": 4246, "restart_count": 13, "start_count": 14}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 10, "h264_decode_error_count": 19, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:40:49+09:00", "last_error_summary": "[h264 @ 0x5561f371cac0] error while decoding MB 43 15, bytestream -5", "last_exit_at": "2026-07-02T11:40:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T11:40:53+09:00", "pid": 4246, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.418, "name": "ffmpeg", "pid": 4246, "rss_mb": 136.418, "state": "S (sleeping)", "threads": 106}, "restart_count": 13, "restart_count_per_hour": 6, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 14}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 218.662, "sample_count": 1451, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 76188, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:40:49+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.987, "last_frame_pushed_at": "2026-07-02T11:40:50+09:00", "last_frame_read_age_sec": 5.225, "last_frame_read_at": "2026-07-02T11:40:49+09:00", "mjpeg": {"age_sec": 4.971, "exists": true, "mtime": "2026-07-02T11:40:50.071028+09:00", "mtime_epoch": 1782960050.071028, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 201193}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:40:55+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1165.184, "name": "celery", "pid": 237, "rss_mb": 1001.551, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 115, "pushed_frames": 115, "read_frames": 114, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.837, "pushed": 3.837, "read": 3.804}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.361, "count": 114, "max_ms": 12.322, "p95_ms": 9.778}, "gstreamer_push": {"avg_ms": 0.863, "count": 115, "max_ms": 1.229, "p95_ms": 0.917}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.492, "count": 114, "max_ms": 75.292, "p95_ms": 65.369}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.246, "count": 114, "max_ms": 99.046, "p95_ms": 92.755}, "yolo11_640_fire_11class": {"avg_ms": 50.766, "count": 114, "max_ms": 86.741, "p95_ms": 64.636}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.301, "count": 114, "max_ms": 104.676, "p95_ms": 91.424}}, "truncated_count": 0}, "processing_total": {"avg_ms": 209.486, "count": 115, "max_ms": 291.247, "p95_ms": 237.442}, "read_interval": {"avg_ms": 218.927, "count": 114, "max_ms": 301.774, "p95_ms": 248.857}, "triton_wait": {"avg_ms": 69.63, "count": 114, "max_ms": 104.689, "p95_ms": 95.049}, "visualize": {"avg_ms": 9.65, "count": 114, "max_ms": 12.81, "p95_ms": 10.96}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.971, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 591}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4480.137, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x56476be194c0] error while decoding MB 8 55, bytestream -7 [in#0/rtsp @ 0x56476bc7ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56476bc7ee00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 11, "h264_decode_error_count": 11, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:42:22+09:00", "last_error_summary": "[h264 @ 0x56476be194c0] error while decoding MB 8 55, bytestream -7 [in#0/rtsp @ 0x56476bc7ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56476bc7ee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T11:40:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.66, "last_started_at": "2026-07-02T11:40:10+09:00", "pid": 4094, "process": {"exists": false, "pid": 4094}, "restart_count": 10, "restart_count_per_hour": 6, "rtsp_timeout_count": 14, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 11}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.186, "sample_count": 1892, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 71036, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:42:21+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.073, "last_frame_pushed_at": "2026-07-02T11:42:21+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-02T11:42:22+09:00", "mjpeg": {"age_sec": 0.058, "exists": true, "mtime": "2026-07-02T11:42:22.012047+09:00", "mtime_epoch": 1782960142.0120466, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 241099}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T11:42:22+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1169.93, "name": "celery", "pid": 267, "rss_mb": 1034.898, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.725, "pushed": 0.725, "read": 0.78}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.821, "count": 13, "max_ms": 5.71, "p95_ms": 5.71}, "gstreamer_push": {"avg_ms": 1.077, "count": 13, "max_ms": 1.325, "p95_ms": 1.325}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.573, "count": 13, "max_ms": 68.89, "p95_ms": 68.89}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.748, "count": 13, "max_ms": 78.592, "p95_ms": 78.592}, "yolo11_640_fire_11class": {"avg_ms": 49.666, "count": 13, "max_ms": 62.924, "p95_ms": 62.924}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.099, "count": 13, "max_ms": 91.361, "p95_ms": 91.361}}, "truncated_count": 0}, "processing_total": {"avg_ms": 115.33, "count": 13, "max_ms": 143.473, "p95_ms": 143.473}, "read_interval": {"avg_ms": 2223.691, "count": 14, "max_ms": 16371.029, "p95_ms": 16371.029}, "triton_wait": {"avg_ms": 62.468, "count": 13, "max_ms": 91.374, "p95_ms": 91.374}, "visualize": {"avg_ms": 5.109, "count": 13, "max_ms": 5.631, "p95_ms": 5.631}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.943, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 591}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4479.113, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T11:42:22+09:00", "last_error_summary": "[h264 @ 0x56476be194c0] error while decoding MB 8 55, bytestream -7 [in#0/rtsp @ 0x56476bc7ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56476bc7ee00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 11, "h264_decode_error_count": 11, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:42:22+09:00", "last_error_summary": "[h264 @ 0x56476be194c0] error while decoding MB 8 55, bytestream -7 [in#0/rtsp @ 0x56476bc7ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56476bc7ee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T11:42:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.739, "last_started_at": "2026-07-02T11:40:10+09:00", "pid": 4094, "process": {"exists": false, "pid": 4094}, "restart_count": 10, "restart_count_per_hour": 6, "rtsp_timeout_count": 14, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 11}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.186, "sample_count": 1892, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 71037, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:42:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T11:42:22+09:00", "last_frame_read_age_sec": 0.142, "last_frame_read_at": "2026-07-02T11:42:22+09:00", "mjpeg": {"age_sec": 0.136, "exists": true, "mtime": "2026-07-02T11:42:22.012047+09:00", "mtime_epoch": 1782960142.0120466, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 241099}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T11:42:22+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1169.93, "name": "celery", "pid": 267, "rss_mb": 1034.898, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.777, "pushed": 0.777, "read": 0.777}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.813, "count": 14, "max_ms": 5.71, "p95_ms": 5.71}, "gstreamer_push": {"avg_ms": 1.09, "count": 14, "max_ms": 1.325, "p95_ms": 1.325}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.833, "count": 14, "max_ms": 68.89, "p95_ms": 68.89}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.838, "count": 14, "max_ms": 78.592, "p95_ms": 78.592}, "yolo11_640_fire_11class": {"avg_ms": 50.92, "count": 14, "max_ms": 67.219, "p95_ms": 67.219}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.906, "count": 14, "max_ms": 91.361, "p95_ms": 91.361}}, "truncated_count": 0}, "processing_total": {"avg_ms": 116.833, "count": 14, "max_ms": 143.473, "p95_ms": 143.473}, "read_interval": {"avg_ms": 2223.691, "count": 14, "max_ms": 16371.029, "p95_ms": 16371.029}, "triton_wait": {"avg_ms": 63.964, "count": 14, "max_ms": 91.374, "p95_ms": 91.374}, "visualize": {"avg_ms": 5.12, "count": 14, "max_ms": 5.631, "p95_ms": 5.631}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.021, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 591}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4479.105, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 11, "h264_decode_error_count": 11, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:42:22+09:00", "last_error_summary": "[h264 @ 0x56476be194c0] error while decoding MB 8 55, bytestream -7 [in#0/rtsp @ 0x56476bc7ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56476bc7ee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T11:42:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.743, "last_started_at": "2026-07-02T11:40:10+09:00", "pid": 4094, "process": {"exists": false, "pid": 4094}, "restart_count": 10, "restart_count_per_hour": 6, "rtsp_timeout_count": 14, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 11}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.186, "sample_count": 1892, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 71037, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:42:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-02T11:42:22+09:00", "last_frame_read_age_sec": 0.147, "last_frame_read_at": "2026-07-02T11:42:22+09:00", "mjpeg": {"age_sec": 0.141, "exists": true, "mtime": "2026-07-02T11:42:22.012047+09:00", "mtime_epoch": 1782960142.0120466, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 241099}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T11:42:22+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1169.93, "name": "celery", "pid": 267, "rss_mb": 1034.898, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.777, "pushed": 0.777, "read": 0.777}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.813, "count": 14, "max_ms": 5.71, "p95_ms": 5.71}, "gstreamer_push": {"avg_ms": 1.09, "count": 14, "max_ms": 1.325, "p95_ms": 1.325}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.833, "count": 14, "max_ms": 68.89, "p95_ms": 68.89}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.838, "count": 14, "max_ms": 78.592, "p95_ms": 78.592}, "yolo11_640_fire_11class": {"avg_ms": 50.92, "count": 14, "max_ms": 67.219, "p95_ms": 67.219}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.906, "count": 14, "max_ms": 91.361, "p95_ms": 91.361}}, "truncated_count": 0}, "processing_total": {"avg_ms": 116.833, "count": 14, "max_ms": 143.473, "p95_ms": 143.473}, "read_interval": {"avg_ms": 2223.691, "count": 14, "max_ms": 16371.029, "p95_ms": 16371.029}, "triton_wait": {"avg_ms": 63.964, "count": 14, "max_ms": 91.374, "p95_ms": 91.374}, "visualize": {"avg_ms": 5.12, "count": 14, "max_ms": 5.631, "p95_ms": 5.631}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.025, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 591}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4491.766, "limit_mb": 8796093022207.996}, "detail": {"pid": 4353, "restart_count": 11, "start_count": 12}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 11, "h264_decode_error_count": 11, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:42:22+09:00", "last_error_summary": "[h264 @ 0x56476be194c0] error while decoding MB 8 55, bytestream -7 [in#0/rtsp @ 0x56476bc7ee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56476bc7ee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T11:42:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T11:42:24+09:00", "pid": 4353, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.461, "name": "ffmpeg", "pid": 4353, "rss_mb": 58.461, "state": "S (sleeping)", "threads": 1}, "restart_count": 11, "restart_count_per_hour": 7, "rtsp_timeout_count": 14, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 12}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.157, "sample_count": 1856, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 71037, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:42:22+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 4.697, "last_frame_pushed_at": "2026-07-02T11:42:22+09:00", "last_frame_read_age_sec": 4.836, "last_frame_read_at": "2026-07-02T11:42:22+09:00", "mjpeg": {"age_sec": 4.682, "exists": true, "mtime": "2026-07-02T11:42:22.160048+09:00", "mtime_epoch": 1782960142.1600482, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 242076}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T11:42:26+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1169.93, "name": "celery", "pid": 267, "rss_mb": 1034.898, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.616, "pushed": 0.616, "read": 0.616}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.813, "count": 14, "max_ms": 5.71, "p95_ms": 5.71}, "gstreamer_push": {"avg_ms": 1.09, "count": 14, "max_ms": 1.325, "p95_ms": 1.325}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.833, "count": 14, "max_ms": 68.89, "p95_ms": 68.89}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.838, "count": 14, "max_ms": 78.592, "p95_ms": 78.592}, "yolo11_640_fire_11class": {"avg_ms": 50.92, "count": 14, "max_ms": 67.219, "p95_ms": 67.219}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.906, "count": 14, "max_ms": 91.361, "p95_ms": 91.361}}, "truncated_count": 0}, "processing_total": {"avg_ms": 116.833, "count": 14, "max_ms": 143.473, "p95_ms": 143.473}, "read_interval": {"avg_ms": 2223.691, "count": 14, "max_ms": 16371.029, "p95_ms": 16371.029}, "triton_wait": {"avg_ms": 63.964, "count": 14, "max_ms": 91.374, "p95_ms": 91.374}, "visualize": {"avg_ms": 5.12, "count": 14, "max_ms": 5.631, "p95_ms": 5.631}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.715, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 639}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4574.301, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55630a3ede80] error while decoding MB 15 21, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 11, "h264_decode_error_count": 20, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:43:02+09:00", "last_error_summary": "[h264 @ 0x55630a3ede80] error while decoding MB 15 21, bytestream -7", "last_exit_at": "2026-07-02T11:40:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.479, "last_started_at": "2026-07-02T11:40:53+09:00", "pid": 4246, "process": {"exists": false, "pid": 4246}, "restart_count": 13, "restart_count_per_hour": 6, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 14}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 244.51, "sample_count": 1380, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 76752, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:43:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.065, "last_frame_pushed_at": "2026-07-02T11:43:02+09:00", "last_frame_read_age_sec": 0.057, "last_frame_read_at": "2026-07-02T11:43:02+09:00", "mjpeg": {"age_sec": 0.048, "exists": true, "mtime": "2026-07-02T11:43:02.472534+09:00", "mtime_epoch": 1782960182.4725337, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199917}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:43:02+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1165.184, "name": "celery", "pid": 237, "rss_mb": 1002.066, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 123, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.105, "pushed": 4.105, "read": 4.105}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.866, "count": 122, "max_ms": 15.052, "p95_ms": 12.724}, "gstreamer_push": {"avg_ms": 0.881, "count": 123, "max_ms": 1.27, "p95_ms": 0.998}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.088, "count": 122, "max_ms": 82.18, "p95_ms": 70.865}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.406, "count": 122, "max_ms": 112.592, "p95_ms": 93.306}, "yolo11_640_fire_11class": {"avg_ms": 48.64, "count": 122, "max_ms": 75.26, "p95_ms": 64.327}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.489, "count": 122, "max_ms": 117.316, "p95_ms": 99.086}}, "truncated_count": 0}, "processing_total": {"avg_ms": 234.69, "count": 123, "max_ms": 370.269, "p95_ms": 297.589}, "read_interval": {"avg_ms": 244.559, "count": 123, "max_ms": 380.352, "p95_ms": 308.309}, "triton_wait": {"avg_ms": 69.641, "count": 122, "max_ms": 117.326, "p95_ms": 102.947}, "visualize": {"avg_ms": 10.91, "count": 122, "max_ms": 16.637, "p95_ms": 13.405}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.963, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 639}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4570.172, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T11:43:02+09:00", "last_error_summary": "[h264 @ 0x55630a3ede80] error while decoding MB 15 21, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 11, "h264_decode_error_count": 20, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:43:02+09:00", "last_error_summary": "[h264 @ 0x55630a3ede80] error while decoding MB 15 21, bytestream -7", "last_exit_at": "2026-07-02T11:43:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.699, "last_started_at": "2026-07-02T11:40:53+09:00", "pid": 4246, "process": {"exists": false, "pid": 4246}, "restart_count": 13, "restart_count_per_hour": 6, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 14}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 245.186, "sample_count": 1380, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 76753, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:43:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T11:43:02+09:00", "last_frame_read_age_sec": 0.276, "last_frame_read_at": "2026-07-02T11:43:02+09:00", "mjpeg": {"age_sec": 0.267, "exists": true, "mtime": "2026-07-02T11:43:02.472534+09:00", "mtime_epoch": 1782960182.4725337, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199917}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:43:02+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1165.184, "name": "celery", "pid": 237, "rss_mb": 1002.066, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.109, "pushed": 4.109, "read": 4.075}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.911, "count": 122, "max_ms": 15.067, "p95_ms": 12.748}, "gstreamer_push": {"avg_ms": 0.881, "count": 123, "max_ms": 1.27, "p95_ms": 0.998}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.058, "count": 122, "max_ms": 82.18, "p95_ms": 70.865}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.16, "count": 122, "max_ms": 112.592, "p95_ms": 92.808}, "yolo11_640_fire_11class": {"avg_ms": 48.611, "count": 122, "max_ms": 75.26, "p95_ms": 64.327}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.403, "count": 122, "max_ms": 117.316, "p95_ms": 99.086}}, "truncated_count": 0}, "processing_total": {"avg_ms": 235.477, "count": 123, "max_ms": 370.269, "p95_ms": 297.589}, "read_interval": {"avg_ms": 245.049, "count": 122, "max_ms": 380.352, "p95_ms": 308.309}, "triton_wait": {"avg_ms": 69.395, "count": 122, "max_ms": 117.326, "p95_ms": 102.947}, "visualize": {"avg_ms": 10.938, "count": 122, "max_ms": 16.637, "p95_ms": 13.405}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.938, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 639}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4570.164, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 11, "h264_decode_error_count": 20, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:43:02+09:00", "last_error_summary": "[h264 @ 0x55630a3ede80] error while decoding MB 15 21, bytestream -7", "last_exit_at": "2026-07-02T11:43:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 129.702, "last_started_at": "2026-07-02T11:40:53+09:00", "pid": 4246, "process": {"exists": false, "pid": 4246}, "restart_count": 13, "restart_count_per_hour": 6, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 14}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 245.186, "sample_count": 1380, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 76753, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:43:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T11:43:02+09:00", "last_frame_read_age_sec": 0.28, "last_frame_read_at": "2026-07-02T11:43:02+09:00", "mjpeg": {"age_sec": 0.271, "exists": true, "mtime": "2026-07-02T11:43:02.472534+09:00", "mtime_epoch": 1782960182.4725337, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199917}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:43:02+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1165.184, "name": "celery", "pid": 237, "rss_mb": 1002.066, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 123, "pushed_frames": 123, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.108, "pushed": 4.108, "read": 4.075}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.911, "count": 122, "max_ms": 15.067, "p95_ms": 12.748}, "gstreamer_push": {"avg_ms": 0.881, "count": 123, "max_ms": 1.27, "p95_ms": 0.998}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.058, "count": 122, "max_ms": 82.18, "p95_ms": 70.865}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.16, "count": 122, "max_ms": 112.592, "p95_ms": 92.808}, "yolo11_640_fire_11class": {"avg_ms": 48.611, "count": 122, "max_ms": 75.26, "p95_ms": 64.327}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.403, "count": 122, "max_ms": 117.316, "p95_ms": 99.086}}, "truncated_count": 0}, "processing_total": {"avg_ms": 235.477, "count": 123, "max_ms": 370.269, "p95_ms": 297.589}, "read_interval": {"avg_ms": 245.049, "count": 122, "max_ms": 380.352, "p95_ms": 308.309}, "triton_wait": {"avg_ms": 69.395, "count": 122, "max_ms": 117.326, "p95_ms": 102.947}, "visualize": {"avg_ms": 10.938, "count": 122, "max_ms": 16.637, "p95_ms": 13.405}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.941, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 1, "tracking_summary": 639}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4670.859, "limit_mb": 8796093022207.996}, "detail": {"pid": 4565, "restart_count": 14, "start_count": 15}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 11, "h264_decode_error_count": 20, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:43:02+09:00", "last_error_summary": "[h264 @ 0x55630a3ede80] error while decoding MB 15 21, bytestream -7", "last_exit_at": "2026-07-02T11:43:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T11:43:05+09:00", "pid": 4565, "process": {"exists": true, "fd_count": 5, "hwm_mb": 144.641, "name": "ffmpeg", "pid": 4565, "rss_mb": 144.641, "state": "S (sleeping)", "threads": 106}, "restart_count": 14, "restart_count_per_hour": 7, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 15}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 245.452, "sample_count": 1356, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 76753, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:43:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.572, "last_frame_pushed_at": "2026-07-02T11:43:02+09:00", "last_frame_read_age_sec": 4.845, "last_frame_read_at": "2026-07-02T11:43:02+09:00", "mjpeg": {"age_sec": 4.549, "exists": true, "mtime": "2026-07-02T11:43:02.760537+09:00", "mtime_epoch": 1782960182.7605371, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198115}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:43:07+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1165.184, "name": "celery", "pid": 237, "rss_mb": 1002.066, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.452, "pushed": 3.452, "read": 3.419}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.891, "count": 103, "max_ms": 15.067, "p95_ms": 12.989}, "gstreamer_push": {"avg_ms": 0.884, "count": 103, "max_ms": 1.27, "p95_ms": 1.005}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.559, "count": 103, "max_ms": 82.18, "p95_ms": 71.256}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.437, "count": 103, "max_ms": 112.592, "p95_ms": 95.272}, "yolo11_640_fire_11class": {"avg_ms": 49.059, "count": 103, "max_ms": 75.26, "p95_ms": 64.327}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.151, "count": 103, "max_ms": 117.316, "p95_ms": 99.086}}, "truncated_count": 0}, "processing_total": {"avg_ms": 237.643, "count": 103, "max_ms": 370.269, "p95_ms": 300.472}, "read_interval": {"avg_ms": 247.23, "count": 102, "max_ms": 380.352, "p95_ms": 310.804}, "triton_wait": {"avg_ms": 70.027, "count": 103, "max_ms": 117.326, "p95_ms": 102.947}, "visualize": {"avg_ms": 11.034, "count": 103, "max_ms": 16.637, "p95_ms": 13.431}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.835, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 643}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4928.258, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T11:43:02+09:00", "last_error_summary": "[h264 @ 0x55630a3ede80] error while decoding MB 15 21, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 11, "h264_decode_error_count": 20, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:43:02+09:00", "last_error_summary": "[h264 @ 0x55630a3ede80] error while decoding MB 15 21, bytestream -7", "last_exit_at": "2026-07-02T11:44:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.782, "last_started_at": "2026-07-02T11:43:05+09:00", "pid": 4565, "process": {"exists": false, "pid": 4565}, "restart_count": 14, "restart_count_per_hour": 7, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 15}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 268.83, "sample_count": 1293, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 77205, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:44:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T11:44:53+09:00", "last_frame_read_age_sec": 0.236, "last_frame_read_at": "2026-07-02T11:44:52+09:00", "mjpeg": {"age_sec": 0.228, "exists": true, "mtime": "2026-07-02T11:44:52.861966+09:00", "mtime_epoch": 1782960292.8619657, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189332}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:44:53+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1170.719, "name": "celery", "pid": 237, "rss_mb": 1089.465, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 136, "pushed_frames": 136, "read_frames": 135, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.557, "pushed": 4.557, "read": 4.523}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.361, "count": 136, "max_ms": 11.039, "p95_ms": 10.135}, "gstreamer_push": {"avg_ms": 0.88, "count": 136, "max_ms": 1.353, "p95_ms": 0.984}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.623, "count": 136, "max_ms": 74.96, "p95_ms": 69.593}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.379, "count": 136, "max_ms": 117.891, "p95_ms": 96.5}, "yolo11_640_fire_11class": {"avg_ms": 49.141, "count": 136, "max_ms": 72.73, "p95_ms": 66.057}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.919, "count": 136, "max_ms": 123.013, "p95_ms": 95.764}}, "truncated_count": 0}, "processing_total": {"avg_ms": 211.214, "count": 136, "max_ms": 267.982, "p95_ms": 247.39}, "read_interval": {"avg_ms": 220.89, "count": 135, "max_ms": 278.676, "p95_ms": 256.105}, "triton_wait": {"avg_ms": 69.771, "count": 136, "max_ms": 123.026, "p95_ms": 98.848}, "visualize": {"avg_ms": 9.899, "count": 136, "max_ms": 13.415, "p95_ms": 11.328}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.847, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 643}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4928.25, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 11, "h264_decode_error_count": 20, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:43:02+09:00", "last_error_summary": "[h264 @ 0x55630a3ede80] error while decoding MB 15 21, bytestream -7", "last_exit_at": "2026-07-02T11:44:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 107.785, "last_started_at": "2026-07-02T11:43:05+09:00", "pid": 4565, "process": {"exists": false, "pid": 4565}, "restart_count": 14, "restart_count_per_hour": 7, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 15}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 268.83, "sample_count": 1293, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 77205, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:44:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T11:44:53+09:00", "last_frame_read_age_sec": 0.24, "last_frame_read_at": "2026-07-02T11:44:52+09:00", "mjpeg": {"age_sec": 0.231, "exists": true, "mtime": "2026-07-02T11:44:52.861966+09:00", "mtime_epoch": 1782960292.8619657, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189332}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:44:53+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1170.719, "name": "celery", "pid": 237, "rss_mb": 1089.465, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 136, "pushed_frames": 136, "read_frames": 135, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.556, "pushed": 4.556, "read": 4.523}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.361, "count": 136, "max_ms": 11.039, "p95_ms": 10.135}, "gstreamer_push": {"avg_ms": 0.88, "count": 136, "max_ms": 1.353, "p95_ms": 0.984}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.623, "count": 136, "max_ms": 74.96, "p95_ms": 69.593}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.379, "count": 136, "max_ms": 117.891, "p95_ms": 96.5}, "yolo11_640_fire_11class": {"avg_ms": 49.141, "count": 136, "max_ms": 72.73, "p95_ms": 66.057}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.919, "count": 136, "max_ms": 123.013, "p95_ms": 95.764}}, "truncated_count": 0}, "processing_total": {"avg_ms": 211.214, "count": 136, "max_ms": 267.982, "p95_ms": 247.39}, "read_interval": {"avg_ms": 220.89, "count": 135, "max_ms": 278.676, "p95_ms": 256.105}, "triton_wait": {"avg_ms": 69.771, "count": 136, "max_ms": 123.026, "p95_ms": 98.848}, "visualize": {"avg_ms": 9.899, "count": 136, "max_ms": 13.415, "p95_ms": 11.328}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.85, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 643}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4884.633, "limit_mb": 8796093022207.996}, "detail": {"pid": 4792, "restart_count": 15, "start_count": 16}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 11, "h264_decode_error_count": 20, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:43:02+09:00", "last_error_summary": "[h264 @ 0x55630a3ede80] error while decoding MB 15 21, bytestream -7", "last_exit_at": "2026-07-02T11:44:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T11:44:55+09:00", "pid": 4792, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.199, "name": "ffmpeg", "pid": 4792, "rss_mb": 139.199, "state": "S (sleeping)", "threads": 106}, "restart_count": 15, "restart_count_per_hour": 8, "rtsp_timeout_count": 4, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 16}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 269.519, "sample_count": 1270, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 77205, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:44:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.41, "last_frame_pushed_at": "2026-07-02T11:44:53+09:00", "last_frame_read_age_sec": 4.643, "last_frame_read_at": "2026-07-02T11:44:52+09:00", "mjpeg": {"age_sec": 4.395, "exists": true, "mtime": "2026-07-02T11:44:53.101969+09:00", "mtime_epoch": 1782960293.1019688, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192062}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:44:57+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1170.719, "name": "celery", "pid": 237, "rss_mb": 1089.465, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 115, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.882, "pushed": 3.882, "read": 3.849}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.421, "count": 115, "max_ms": 10.9, "p95_ms": 10.135}, "gstreamer_push": {"avg_ms": 0.884, "count": 116, "max_ms": 1.353, "p95_ms": 0.99}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.274, "count": 115, "max_ms": 74.96, "p95_ms": 71.124}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.088, "count": 115, "max_ms": 117.891, "p95_ms": 96.5}, "yolo11_640_fire_11class": {"avg_ms": 48.739, "count": 115, "max_ms": 72.6, "p95_ms": 66.057}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.277, "count": 115, "max_ms": 108.053, "p95_ms": 95.764}}, "truncated_count": 0}, "processing_total": {"avg_ms": 211.749, "count": 116, "max_ms": 267.982, "p95_ms": 247.39}, "read_interval": {"avg_ms": 221.395, "count": 115, "max_ms": 278.676, "p95_ms": 256.105}, "triton_wait": {"avg_ms": 69.25, "count": 115, "max_ms": 117.904, "p95_ms": 98.848}, "visualize": {"avg_ms": 9.959, "count": 115, "max_ms": 13.415, "p95_ms": 11.337}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.88, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 646}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4568.234, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x558200b90000] error while decoding MB 60 42, bytestream -14 [in#0/rtsp @ 0x5581ffdcbe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5581ffdcbe00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 12, "h264_decode_error_count": 21, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:46:53+09:00", "last_error_summary": "[h264 @ 0x558200b90000] error while decoding MB 60 42, bytestream -14 [in#0/rtsp @ 0x5581ffdcbe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5581ffdcbe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T11:44:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 117.879, "last_started_at": "2026-07-02T11:44:55+09:00", "pid": 4792, "process": {"exists": false, "pid": 4792}, "restart_count": 15, "restart_count_per_hour": 8, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 16}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.047, "sample_count": 1094, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 77542, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:46:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-02T11:46:53+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-02T11:46:53+09:00", "mjpeg": {"age_sec": 0.038, "exists": true, "mtime": "2026-07-02T11:46:53.335673+09:00", "mtime_epoch": 1782960413.335673, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200805}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:46:53+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1170.719, "name": "celery", "pid": 237, "rss_mb": 1036.984, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 8, "pushed_frames": 8, "read_frames": 9, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.679, "pushed": 0.679, "read": 0.764}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 9.146, "count": 8, "max_ms": 12.421, "p95_ms": 12.421}, "gstreamer_push": {"avg_ms": 0.856, "count": 8, "max_ms": 0.894, "p95_ms": 0.894}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.648, "count": 8, "max_ms": 66.907, "p95_ms": 66.907}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.271, "count": 8, "max_ms": 95.882, "p95_ms": 95.882}, "yolo11_640_fire_11class": {"avg_ms": 51.955, "count": 8, "max_ms": 66.402, "p95_ms": 66.402}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 69.007, "count": 8, "max_ms": 96.703, "p95_ms": 96.703}}, "truncated_count": 0}, "processing_total": {"avg_ms": 217.604, "count": 8, "max_ms": 264.164, "p95_ms": 264.164}, "read_interval": {"avg_ms": 3364.258, "count": 9, "max_ms": 18550.773, "p95_ms": 18550.773}, "triton_wait": {"avg_ms": 70.657, "count": 8, "max_ms": 96.718, "p95_ms": 96.718}, "visualize": {"avg_ms": 10.043, "count": 8, "max_ms": 11.27, "p95_ms": 11.27}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 11.781, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 646}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4573.555, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T11:46:53+09:00", "last_error_summary": "[h264 @ 0x558200b90000] error while decoding MB 60 42, bytestream -14 [in#0/rtsp @ 0x5581ffdcbe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5581ffdcbe00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 12, "h264_decode_error_count": 21, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:46:53+09:00", "last_error_summary": "[h264 @ 0x558200b90000] error while decoding MB 60 42, bytestream -14 [in#0/rtsp @ 0x5581ffdcbe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5581ffdcbe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T11:46:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 118.066, "last_started_at": "2026-07-02T11:44:55+09:00", "pid": 4792, "process": {"exists": false, "pid": 4792}, "restart_count": 15, "restart_count_per_hour": 8, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 16}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.047, "sample_count": 1094, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 77543, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:46:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T11:46:53+09:00", "last_frame_read_age_sec": 0.241, "last_frame_read_at": "2026-07-02T11:46:53+09:00", "mjpeg": {"age_sec": 0.226, "exists": true, "mtime": "2026-07-02T11:46:53.335673+09:00", "mtime_epoch": 1782960413.335673, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200805}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:46:53+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1170.719, "name": "celery", "pid": 237, "rss_mb": 1036.984, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 9, "pushed_frames": 9, "read_frames": 9, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.752, "pushed": 0.752, "read": 0.752}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 9.032, "count": 9, "max_ms": 12.421, "p95_ms": 12.421}, "gstreamer_push": {"avg_ms": 0.853, "count": 9, "max_ms": 0.894, "p95_ms": 0.894}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.685, "count": 9, "max_ms": 66.907, "p95_ms": 66.907}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.114, "count": 9, "max_ms": 95.882, "p95_ms": 95.882}, "yolo11_640_fire_11class": {"avg_ms": 53.13, "count": 9, "max_ms": 66.402, "p95_ms": 66.402}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.649, "count": 9, "max_ms": 96.703, "p95_ms": 96.703}}, "truncated_count": 0}, "processing_total": {"avg_ms": 219.767, "count": 9, "max_ms": 264.164, "p95_ms": 264.164}, "read_interval": {"avg_ms": 3364.258, "count": 9, "max_ms": 18550.773, "p95_ms": 18550.773}, "triton_wait": {"avg_ms": 73.117, "count": 9, "max_ms": 96.718, "p95_ms": 96.718}, "visualize": {"avg_ms": 10.001, "count": 9, "max_ms": 11.27, "p95_ms": 11.27}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 11.969, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 646}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4573.547, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 12, "h264_decode_error_count": 21, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:46:53+09:00", "last_error_summary": "[h264 @ 0x558200b90000] error while decoding MB 60 42, bytestream -14 [in#0/rtsp @ 0x5581ffdcbe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5581ffdcbe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T11:46:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 118.072, "last_started_at": "2026-07-02T11:44:55+09:00", "pid": 4792, "process": {"exists": false, "pid": 4792}, "restart_count": 15, "restart_count_per_hour": 8, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 16}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.047, "sample_count": 1094, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 77543, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:46:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-02T11:46:53+09:00", "last_frame_read_age_sec": 0.246, "last_frame_read_at": "2026-07-02T11:46:53+09:00", "mjpeg": {"age_sec": 0.232, "exists": true, "mtime": "2026-07-02T11:46:53.335673+09:00", "mtime_epoch": 1782960413.335673, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200805}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:46:53+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1170.719, "name": "celery", "pid": 237, "rss_mb": 1036.984, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 9, "pushed_frames": 9, "read_frames": 9, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.752, "pushed": 0.752, "read": 0.752}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 9.032, "count": 9, "max_ms": 12.421, "p95_ms": 12.421}, "gstreamer_push": {"avg_ms": 0.853, "count": 9, "max_ms": 0.894, "p95_ms": 0.894}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.685, "count": 9, "max_ms": 66.907, "p95_ms": 66.907}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.114, "count": 9, "max_ms": 95.882, "p95_ms": 95.882}, "yolo11_640_fire_11class": {"avg_ms": 53.13, "count": 9, "max_ms": 66.402, "p95_ms": 66.402}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.649, "count": 9, "max_ms": 96.703, "p95_ms": 96.703}}, "truncated_count": 0}, "processing_total": {"avg_ms": 219.767, "count": 9, "max_ms": 264.164, "p95_ms": 264.164}, "read_interval": {"avg_ms": 3364.258, "count": 9, "max_ms": 18550.773, "p95_ms": 18550.773}, "triton_wait": {"avg_ms": 73.117, "count": 9, "max_ms": 96.718, "p95_ms": 96.718}, "visualize": {"avg_ms": 10.001, "count": 9, "max_ms": 11.27, "p95_ms": 11.27}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 11.974, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 646}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4588.883, "limit_mb": 8796093022207.996}, "detail": {"pid": 4959, "restart_count": 16, "start_count": 17}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 12, "h264_decode_error_count": 21, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:46:53+09:00", "last_error_summary": "[h264 @ 0x558200b90000] error while decoding MB 60 42, bytestream -14 [in#0/rtsp @ 0x5581ffdcbe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5581ffdcbe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T11:46:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T11:46:55+09:00", "pid": 4959, "process": {"exists": true, "fd_count": 4, "hwm_mb": 43.496, "name": "ffmpeg", "pid": 4959, "rss_mb": 43.496, "state": "S (sleeping)", "threads": 1}, "restart_count": 16, "restart_count_per_hour": 9, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 17}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 272.863, "sample_count": 1074, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 77543, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:46:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.184, "last_frame_pushed_at": "2026-07-02T11:46:53+09:00", "last_frame_read_age_sec": 4.423, "last_frame_read_at": "2026-07-02T11:46:53+09:00", "mjpeg": {"age_sec": 4.168, "exists": true, "mtime": "2026-07-02T11:46:53.575676+09:00", "mtime_epoch": 1782960413.5756764, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200764}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:46:57+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1170.719, "name": "celery", "pid": 237, "rss_mb": 1036.984, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 9, "pushed_frames": 9, "read_frames": 9, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.557, "pushed": 0.557, "read": 0.557}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 9.032, "count": 9, "max_ms": 12.421, "p95_ms": 12.421}, "gstreamer_push": {"avg_ms": 0.853, "count": 9, "max_ms": 0.894, "p95_ms": 0.894}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.685, "count": 9, "max_ms": 66.907, "p95_ms": 66.907}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.114, "count": 9, "max_ms": 95.882, "p95_ms": 95.882}, "yolo11_640_fire_11class": {"avg_ms": 53.13, "count": 9, "max_ms": 66.402, "p95_ms": 66.402}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.649, "count": 9, "max_ms": 96.703, "p95_ms": 96.703}}, "truncated_count": 0}, "processing_total": {"avg_ms": 219.767, "count": 9, "max_ms": 264.164, "p95_ms": 264.164}, "read_interval": {"avg_ms": 3364.258, "count": 9, "max_ms": 18550.773, "p95_ms": 18550.773}, "triton_wait": {"avg_ms": 73.117, "count": 9, "max_ms": 96.718, "p95_ms": 96.718}, "visualize": {"avg_ms": 10.001, "count": 9, "max_ms": 11.27, "p95_ms": 11.27}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 16.151, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 647}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4535.156, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55e1f2ec4680] error while decoding MB 69 33, bytestream -28", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 13, "h264_decode_error_count": 22, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:48:10+09:00", "last_error_summary": "[h264 @ 0x55e1f2ec4680] error while decoding MB 69 33, bytestream -28", "last_exit_at": "2026-07-02T11:46:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 74.398, "last_started_at": "2026-07-02T11:46:55+09:00", "pid": 4959, "process": {"exists": false, "pid": 4959}, "restart_count": 16, "restart_count_per_hour": 9, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 17}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.465, "sample_count": 916, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 77681, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:48:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-02T11:48:10+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-02T11:48:10+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-02T11:48:10.100825+09:00", "mtime_epoch": 1782960490.1008253, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198690}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:48:10+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1170.719, "name": "celery", "pid": 237, "rss_mb": 1005.344, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 81, "pushed_frames": 81, "read_frames": 82, "select_timeouts": 8, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.701, "pushed": 2.701, "read": 2.734}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 10.275, "count": 81, "max_ms": 13.158, "p95_ms": 11.665}, "gstreamer_push": {"avg_ms": 0.877, "count": 81, "max_ms": 1.134, "p95_ms": 0.936}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.374, "count": 81, "max_ms": 73.529, "p95_ms": 65.734}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.78, "count": 81, "max_ms": 115.01, "p95_ms": 108.354}, "yolo11_640_fire_11class": {"avg_ms": 50.4, "count": 81, "max_ms": 73.947, "p95_ms": 64.469}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.682, "count": 81, "max_ms": 106.703, "p95_ms": 98.846}}, "truncated_count": 0}, "processing_total": {"avg_ms": 248.572, "count": 81, "max_ms": 304.434, "p95_ms": 289.972}, "read_interval": {"avg_ms": 368.303, "count": 82, "max_ms": 5449.976, "p95_ms": 309.339}, "triton_wait": {"avg_ms": 74.001, "count": 81, "max_ms": 115.024, "p95_ms": 108.367}, "visualize": {"avg_ms": 11.171, "count": 81, "max_ms": 13.642, "p95_ms": 12.296}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 647}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4545.609, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T11:48:10+09:00", "last_error_summary": "[h264 @ 0x55e1f2ec4680] error while decoding MB 69 33, bytestream -28", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 13, "h264_decode_error_count": 22, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:48:10+09:00", "last_error_summary": "[h264 @ 0x55e1f2ec4680] error while decoding MB 69 33, bytestream -28", "last_exit_at": "2026-07-02T11:48:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 74.598, "last_started_at": "2026-07-02T11:46:55+09:00", "pid": 4959, "process": {"exists": false, "pid": 4959}, "restart_count": 16, "restart_count_per_hour": 9, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 17}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.465, "sample_count": 916, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 77682, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:48:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T11:48:10+09:00", "last_frame_read_age_sec": 0.251, "last_frame_read_at": "2026-07-02T11:48:10+09:00", "mjpeg": {"age_sec": 0.24, "exists": true, "mtime": "2026-07-02T11:48:10.100825+09:00", "mtime_epoch": 1782960490.1008253, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198690}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:48:10+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1170.719, "name": "celery", "pid": 237, "rss_mb": 1005.344, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 82, "pushed_frames": 82, "read_frames": 81, "select_timeouts": 8, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.74, "pushed": 2.74, "read": 2.707}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 10.258, "count": 81, "max_ms": 13.158, "p95_ms": 11.665}, "gstreamer_push": {"avg_ms": 0.877, "count": 82, "max_ms": 1.134, "p95_ms": 0.936}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.435, "count": 81, "max_ms": 73.529, "p95_ms": 65.734}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.572, "count": 81, "max_ms": 115.01, "p95_ms": 108.354}, "yolo11_640_fire_11class": {"avg_ms": 50.342, "count": 81, "max_ms": 73.947, "p95_ms": 64.469}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.69, "count": 81, "max_ms": 106.703, "p95_ms": 98.846}}, "truncated_count": 0}, "processing_total": {"avg_ms": 248.551, "count": 82, "max_ms": 304.434, "p95_ms": 289.972}, "read_interval": {"avg_ms": 369.652, "count": 81, "max_ms": 5449.976, "p95_ms": 309.339}, "triton_wait": {"avg_ms": 73.793, "count": 81, "max_ms": 115.024, "p95_ms": 108.367}, "visualize": {"avg_ms": 11.165, "count": 81, "max_ms": 13.642, "p95_ms": 12.296}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.925, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 647}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4545.602, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 13, "h264_decode_error_count": 22, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:48:10+09:00", "last_error_summary": "[h264 @ 0x55e1f2ec4680] error while decoding MB 69 33, bytestream -28", "last_exit_at": "2026-07-02T11:48:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 74.601, "last_started_at": "2026-07-02T11:46:55+09:00", "pid": 4959, "process": {"exists": false, "pid": 4959}, "restart_count": 16, "restart_count_per_hour": 9, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 17}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 274.465, "sample_count": 916, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 77682, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:48:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T11:48:10+09:00", "last_frame_read_age_sec": 0.254, "last_frame_read_at": "2026-07-02T11:48:10+09:00", "mjpeg": {"age_sec": 0.243, "exists": true, "mtime": "2026-07-02T11:48:10.100825+09:00", "mtime_epoch": 1782960490.1008253, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198690}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:48:10+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1170.719, "name": "celery", "pid": 237, "rss_mb": 1005.344, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 82, "pushed_frames": 82, "read_frames": 81, "select_timeouts": 8, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.74, "pushed": 2.74, "read": 2.706}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 10.258, "count": 81, "max_ms": 13.158, "p95_ms": 11.665}, "gstreamer_push": {"avg_ms": 0.877, "count": 82, "max_ms": 1.134, "p95_ms": 0.936}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.435, "count": 81, "max_ms": 73.529, "p95_ms": 65.734}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.572, "count": 81, "max_ms": 115.01, "p95_ms": 108.354}, "yolo11_640_fire_11class": {"avg_ms": 50.342, "count": 81, "max_ms": 73.947, "p95_ms": 64.469}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.69, "count": 81, "max_ms": 106.703, "p95_ms": 98.846}}, "truncated_count": 0}, "processing_total": {"avg_ms": 248.551, "count": 82, "max_ms": 304.434, "p95_ms": 289.972}, "read_interval": {"avg_ms": 369.652, "count": 81, "max_ms": 5449.976, "p95_ms": 309.339}, "triton_wait": {"avg_ms": 73.793, "count": 81, "max_ms": 115.024, "p95_ms": 108.367}, "visualize": {"avg_ms": 11.165, "count": 81, "max_ms": 13.642, "p95_ms": 12.296}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.928, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 2, "tracking_summary": 647}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4644.785, "limit_mb": 8796093022207.996}, "detail": {"pid": 5066, "restart_count": 17, "start_count": 18}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 13, "h264_decode_error_count": 22, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:48:10+09:00", "last_error_summary": "[h264 @ 0x55e1f2ec4680] error while decoding MB 69 33, bytestream -28", "last_exit_at": "2026-07-02T11:48:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T11:48:12+09:00", "pid": 5066, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.93, "name": "ffmpeg", "pid": 5066, "rss_mb": 136.93, "state": "S (sleeping)", "threads": 106}, "restart_count": 17, "restart_count_per_hour": 10, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 18}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 275.19, "sample_count": 896, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 77682, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:48:10+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.449, "last_frame_pushed_at": "2026-07-02T11:48:10+09:00", "last_frame_read_age_sec": 4.697, "last_frame_read_at": "2026-07-02T11:48:10+09:00", "mjpeg": {"age_sec": 4.43, "exists": true, "mtime": "2026-07-02T11:48:10.356829+09:00", "mtime_epoch": 1782960490.3568292, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198770}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:48:14+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1170.719, "name": "celery", "pid": 237, "rss_mb": 1005.344, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 73, "pushed_frames": 73, "read_frames": 72, "select_timeouts": 6, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.448, "pushed": 2.448, "read": 2.415}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 10.207, "count": 73, "max_ms": 13.158, "p95_ms": 11.665}, "gstreamer_push": {"avg_ms": 0.877, "count": 73, "max_ms": 1.134, "p95_ms": 0.959}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.384, "count": 73, "max_ms": 73.529, "p95_ms": 65.881}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 72.097, "count": 73, "max_ms": 115.01, "p95_ms": 110.929}, "yolo11_640_fire_11class": {"avg_ms": 49.942, "count": 73, "max_ms": 73.947, "p95_ms": 64.02}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.86, "count": 73, "max_ms": 105.882, "p95_ms": 96.125}}, "truncated_count": 0}, "processing_total": {"avg_ms": 246.679, "count": 73, "max_ms": 304.434, "p95_ms": 293.713}, "read_interval": {"avg_ms": 352.268, "count": 72, "max_ms": 5449.976, "p95_ms": 309.339}, "triton_wait": {"avg_ms": 73.22, "count": 73, "max_ms": 115.024, "p95_ms": 110.943}, "visualize": {"avg_ms": 11.091, "count": 73, "max_ms": 13.642, "p95_ms": 12.633}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.817, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 14, "tracking_summary": 613}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4846.375, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x563617fbe280] error while decoding MB 52 67, bytestream -11 [in#0/rtsp @ 0x563617df3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563617df3e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 12, "h264_decode_error_count": 12, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:48:38+09:00", "last_error_summary": "[h264 @ 0x563617fbe280] error while decoding MB 52 67, bytestream -11 [in#0/rtsp @ 0x563617df3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563617df3e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T11:42:24+09:00", "last_returncode": 0, "last_stable_duration_sec": 374.088, "last_started_at": "2026-07-02T11:42:24+09:00", "pid": 4353, "process": {"exists": false, "pid": 4353}, "restart_count": 11, "restart_count_per_hour": 7, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 12}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 178.024, "sample_count": 2143, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 73586, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:48:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.079, "last_frame_pushed_at": "2026-07-02T11:48:38+09:00", "last_frame_read_age_sec": 0.067, "last_frame_read_at": "2026-07-02T11:48:38+09:00", "mjpeg": {"age_sec": 0.06, "exists": true, "mtime": "2026-07-02T11:48:38.869268+09:00", "mtime_epoch": 1782960518.8692682, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 250215}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T11:48:38+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 999.156, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 99, "pushed_frames": 99, "read_frames": 100, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.3, "pushed": 3.3, "read": 3.333}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.112, "count": 100, "max_ms": 6.46, "p95_ms": 5.72}, "gstreamer_push": {"avg_ms": 1.153, "count": 99, "max_ms": 1.663, "p95_ms": 1.386}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.207, "count": 100, "max_ms": 76.425, "p95_ms": 55.386}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.061, "count": 100, "max_ms": 107.897, "p95_ms": 83.696}, "yolo11_640_fire_11class": {"avg_ms": 48.799, "count": 100, "max_ms": 77.152, "p95_ms": 57.849}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.338, "count": 100, "max_ms": 107.909, "p95_ms": 81.907}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.924, "count": 99, "max_ms": 137.506, "p95_ms": 130.148}, "read_interval": {"avg_ms": 300.532, "count": 100, "max_ms": 10163.051, "p95_ms": 142.761}, "triton_wait": {"avg_ms": 59.632, "count": 100, "max_ms": 107.919, "p95_ms": 83.708}, "visualize": {"avg_ms": 4.6, "count": 100, "max_ms": 6.342, "p95_ms": 5.59}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 14, "tracking_summary": 613}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4842.746, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T11:48:38+09:00", "last_error_summary": "[h264 @ 0x563617fbe280] error while decoding MB 52 67, bytestream -11 [in#0/rtsp @ 0x563617df3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563617df3e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 12, "h264_decode_error_count": 12, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:48:38+09:00", "last_error_summary": "[h264 @ 0x563617fbe280] error while decoding MB 52 67, bytestream -11 [in#0/rtsp @ 0x563617df3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563617df3e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T11:48:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 374.11, "last_started_at": "2026-07-02T11:42:24+09:00", "pid": 4353, "process": {"exists": false, "pid": 4353}, "restart_count": 11, "restart_count_per_hour": 7, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 12}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 178.024, "sample_count": 2144, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 73586, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:48:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-02T11:48:38+09:00", "last_frame_read_age_sec": 0.088, "last_frame_read_at": "2026-07-02T11:48:38+09:00", "mjpeg": {"age_sec": 0.081, "exists": true, "mtime": "2026-07-02T11:48:38.869268+09:00", "mtime_epoch": 1782960518.8692682, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 250215}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T11:48:38+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 999.156, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.343, "pushed": 3.343, "read": 3.31}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.112, "count": 100, "max_ms": 6.46, "p95_ms": 5.72}, "gstreamer_push": {"avg_ms": 1.155, "count": 100, "max_ms": 1.663, "p95_ms": 1.386}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.207, "count": 100, "max_ms": 76.425, "p95_ms": 55.386}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.061, "count": 100, "max_ms": 107.897, "p95_ms": 83.696}, "yolo11_640_fire_11class": {"avg_ms": 48.799, "count": 100, "max_ms": 77.152, "p95_ms": 57.849}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.338, "count": 100, "max_ms": 107.909, "p95_ms": 81.907}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.716, "count": 100, "max_ms": 137.506, "p95_ms": 128.662}, "read_interval": {"avg_ms": 302.348, "count": 99, "max_ms": 10163.051, "p95_ms": 145.106}, "triton_wait": {"avg_ms": 59.632, "count": 100, "max_ms": 107.919, "p95_ms": 83.708}, "visualize": {"avg_ms": 4.6, "count": 100, "max_ms": 6.342, "p95_ms": 5.59}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.911, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 14, "tracking_summary": 613}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4842.715, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 12, "h264_decode_error_count": 12, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:48:38+09:00", "last_error_summary": "[h264 @ 0x563617fbe280] error while decoding MB 52 67, bytestream -11 [in#0/rtsp @ 0x563617df3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563617df3e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T11:48:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 374.117, "last_started_at": "2026-07-02T11:42:24+09:00", "pid": 4353, "process": {"exists": false, "pid": 4353}, "restart_count": 11, "restart_count_per_hour": 7, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 12}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 178.024, "sample_count": 2144, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 73586, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:48:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.017, "last_frame_pushed_at": "2026-07-02T11:48:38+09:00", "last_frame_read_age_sec": 0.096, "last_frame_read_at": "2026-07-02T11:48:38+09:00", "mjpeg": {"age_sec": 0.089, "exists": true, "mtime": "2026-07-02T11:48:38.869268+09:00", "mtime_epoch": 1782960518.8692682, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 250215}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T11:48:38+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 999.156, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 100, "pushed_frames": 100, "read_frames": 99, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.342, "pushed": 3.342, "read": 3.309}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.112, "count": 100, "max_ms": 6.46, "p95_ms": 5.72}, "gstreamer_push": {"avg_ms": 1.155, "count": 100, "max_ms": 1.663, "p95_ms": 1.386}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.207, "count": 100, "max_ms": 76.425, "p95_ms": 55.386}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.061, "count": 100, "max_ms": 107.897, "p95_ms": 83.696}, "yolo11_640_fire_11class": {"avg_ms": 48.799, "count": 100, "max_ms": 77.152, "p95_ms": 57.849}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.338, "count": 100, "max_ms": 107.909, "p95_ms": 81.907}}, "truncated_count": 0}, "processing_total": {"avg_ms": 97.716, "count": 100, "max_ms": 137.506, "p95_ms": 128.662}, "read_interval": {"avg_ms": 302.348, "count": 99, "max_ms": 10163.051, "p95_ms": 145.106}, "triton_wait": {"avg_ms": 59.632, "count": 100, "max_ms": 107.919, "p95_ms": 83.708}, "visualize": {"avg_ms": 4.6, "count": 100, "max_ms": 6.342, "p95_ms": 5.59}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.918, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 14, "tracking_summary": 613}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5008.73, "limit_mb": 8796093022207.996}, "detail": {"pid": 5233, "restart_count": 12, "start_count": 13}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 12, "h264_decode_error_count": 12, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:48:38+09:00", "last_error_summary": "[h264 @ 0x563617fbe280] error while decoding MB 52 67, bytestream -11 [in#0/rtsp @ 0x563617df3e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x563617df3e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T11:48:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T11:48:41+09:00", "pid": 5233, "process": {"exists": true, "fd_count": 5, "hwm_mb": 195.02, "name": "ffmpeg", "pid": 5233, "rss_mb": 195.02, "state": "S (sleeping)", "threads": 106}, "restart_count": 12, "restart_count_per_hour": 8, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 13}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 178.613, "sample_count": 2103, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 73586, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:48:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 4.479, "last_frame_pushed_at": "2026-07-02T11:48:38+09:00", "last_frame_read_age_sec": 4.558, "last_frame_read_at": "2026-07-02T11:48:38+09:00", "mjpeg": {"age_sec": 4.463, "exists": true, "mtime": "2026-07-02T11:48:38.957270+09:00", "mtime_epoch": 1782960518.9572697, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 250158}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T11:48:43+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 999.156, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 63, "pushed_frames": 63, "read_frames": 62, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.102, "pushed": 2.102, "read": 2.068}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.75, "count": 62, "max_ms": 6.148, "p95_ms": 5.72}, "gstreamer_push": {"avg_ms": 1.199, "count": 63, "max_ms": 1.663, "p95_ms": 1.397}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.235, "count": 62, "max_ms": 76.425, "p95_ms": 57.74}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.361, "count": 62, "max_ms": 107.897, "p95_ms": 84.59}, "yolo11_640_fire_11class": {"avg_ms": 49.889, "count": 62, "max_ms": 77.152, "p95_ms": 58.871}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.314, "count": 62, "max_ms": 107.909, "p95_ms": 83.633}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.562, "count": 63, "max_ms": 137.506, "p95_ms": 130.148}, "read_interval": {"avg_ms": 411.634, "count": 62, "max_ms": 10163.051, "p95_ms": 146.379}, "triton_wait": {"avg_ms": 62.168, "count": 62, "max_ms": 107.919, "p95_ms": 84.602}, "visualize": {"avg_ms": 4.307, "count": 62, "max_ms": 6.342, "p95_ms": 5.563}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.976, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 650}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5023.719, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55c82a096900] error while decoding MB 30 10, bytestream -24", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 14, "h264_decode_error_count": 23, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:50:12+09:00", "last_error_summary": "[h264 @ 0x55c82a096900] error while decoding MB 30 10, bytestream -24", "last_exit_at": "2026-07-02T11:48:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 120.128, "last_started_at": "2026-07-02T11:48:12+09:00", "pid": 5066, "process": {"exists": false, "pid": 5066}, "restart_count": 17, "restart_count_per_hour": 10, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 18}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 340.74, "sample_count": 799, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 78073, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:50:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-02T11:50:12+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-02T11:50:12+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-02T11:50:12.874753+09:00", "mtime_epoch": 1782960612.8747528, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199110}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:50:12+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1366.539, "name": "celery", "pid": 237, "rss_mb": 1316.672, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 104, "pushed_frames": 104, "read_frames": 104, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.482, "pushed": 3.482, "read": 3.482}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 12.256, "count": 103, "max_ms": 17.97, "p95_ms": 14.427}, "gstreamer_push": {"avg_ms": 0.875, "count": 104, "max_ms": 0.994, "p95_ms": 0.933}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.09, "count": 103, "max_ms": 77.602, "p95_ms": 70.602}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.517, "count": 103, "max_ms": 107.796, "p95_ms": 94.159}, "yolo11_640_fire_11class": {"avg_ms": 48.863, "count": 103, "max_ms": 71.743, "p95_ms": 62.767}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.4, "count": 103, "max_ms": 111.503, "p95_ms": 99.602}}, "truncated_count": 0}, "processing_total": {"avg_ms": 279.916, "count": 104, "max_ms": 330.147, "p95_ms": 313.371}, "read_interval": {"avg_ms": 289.428, "count": 104, "max_ms": 339.483, "p95_ms": 322.965}, "triton_wait": {"avg_ms": 70.147, "count": 103, "max_ms": 111.515, "p95_ms": 101.882}, "visualize": {"avg_ms": 12.38, "count": 103, "max_ms": 17.117, "p95_ms": 13.775}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.867, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 650}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5021.703, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T11:50:12+09:00", "last_error_summary": "[h264 @ 0x55c82a096900] error while decoding MB 30 10, bytestream -24", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 14, "h264_decode_error_count": 23, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:50:12+09:00", "last_error_summary": "[h264 @ 0x55c82a096900] error while decoding MB 30 10, bytestream -24", "last_exit_at": "2026-07-02T11:50:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 120.362, "last_started_at": "2026-07-02T11:48:12+09:00", "pid": 5066, "process": {"exists": false, "pid": 5066}, "restart_count": 17, "restart_count_per_hour": 10, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 18}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 340.74, "sample_count": 798, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 78074, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:50:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T11:50:13+09:00", "last_frame_read_age_sec": 0.283, "last_frame_read_at": "2026-07-02T11:50:12+09:00", "mjpeg": {"age_sec": 0.274, "exists": true, "mtime": "2026-07-02T11:50:12.874753+09:00", "mtime_epoch": 1782960612.8747528, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199110}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:50:13+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1366.539, "name": "celery", "pid": 237, "rss_mb": 1316.672, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 104, "pushed_frames": 104, "read_frames": 103, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.485, "pushed": 3.485, "read": 3.451}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 12.281, "count": 103, "max_ms": 17.97, "p95_ms": 14.427}, "gstreamer_push": {"avg_ms": 0.876, "count": 104, "max_ms": 0.994, "p95_ms": 0.933}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.056, "count": 103, "max_ms": 77.602, "p95_ms": 70.602}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.505, "count": 103, "max_ms": 107.796, "p95_ms": 94.159}, "yolo11_640_fire_11class": {"avg_ms": 48.823, "count": 103, "max_ms": 71.743, "p95_ms": 62.767}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.438, "count": 103, "max_ms": 111.503, "p95_ms": 99.602}}, "truncated_count": 0}, "processing_total": {"avg_ms": 279.871, "count": 104, "max_ms": 330.147, "p95_ms": 313.371}, "read_interval": {"avg_ms": 289.403, "count": 103, "max_ms": 339.483, "p95_ms": 322.965}, "triton_wait": {"avg_ms": 70.146, "count": 103, "max_ms": 111.515, "p95_ms": 101.882}, "visualize": {"avg_ms": 12.397, "count": 103, "max_ms": 17.117, "p95_ms": 13.775}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.844, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 650}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5023.195, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 14, "h264_decode_error_count": 23, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:50:12+09:00", "last_error_summary": "[h264 @ 0x55c82a096900] error while decoding MB 30 10, bytestream -24", "last_exit_at": "2026-07-02T11:50:13+09:00", "last_returncode": 0, "last_stable_duration_sec": 120.366, "last_started_at": "2026-07-02T11:48:12+09:00", "pid": 5066, "process": {"exists": false, "pid": 5066}, "restart_count": 17, "restart_count_per_hour": 10, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 18}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 340.74, "sample_count": 798, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 78074, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:50:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T11:50:13+09:00", "last_frame_read_age_sec": 0.286, "last_frame_read_at": "2026-07-02T11:50:12+09:00", "mjpeg": {"age_sec": 0.277, "exists": true, "mtime": "2026-07-02T11:50:12.874753+09:00", "mtime_epoch": 1782960612.8747528, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199110}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:50:13+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1366.539, "name": "celery", "pid": 237, "rss_mb": 1316.672, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 104, "pushed_frames": 104, "read_frames": 103, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.484, "pushed": 3.484, "read": 3.451}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 12.281, "count": 103, "max_ms": 17.97, "p95_ms": 14.427}, "gstreamer_push": {"avg_ms": 0.876, "count": 104, "max_ms": 0.994, "p95_ms": 0.933}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.056, "count": 103, "max_ms": 77.602, "p95_ms": 70.602}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.505, "count": 103, "max_ms": 107.796, "p95_ms": 94.159}, "yolo11_640_fire_11class": {"avg_ms": 48.823, "count": 103, "max_ms": 71.743, "p95_ms": 62.767}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.438, "count": 103, "max_ms": 111.503, "p95_ms": 99.602}}, "truncated_count": 0}, "processing_total": {"avg_ms": 279.871, "count": 104, "max_ms": 330.147, "p95_ms": 313.371}, "read_interval": {"avg_ms": 289.403, "count": 103, "max_ms": 339.483, "p95_ms": 322.965}, "triton_wait": {"avg_ms": 70.146, "count": 103, "max_ms": 111.515, "p95_ms": 101.882}, "visualize": {"avg_ms": 12.397, "count": 103, "max_ms": 17.117, "p95_ms": 13.775}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.848, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 650}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5133.969, "limit_mb": 8796093022207.996}, "detail": {"pid": 5445, "restart_count": 18, "start_count": 19}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 14, "h264_decode_error_count": 23, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:50:12+09:00", "last_error_summary": "[h264 @ 0x55c82a096900] error while decoding MB 30 10, bytestream -24", "last_exit_at": "2026-07-02T11:50:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T11:50:15+09:00", "pid": 5445, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.555, "name": "ffmpeg", "pid": 5445, "rss_mb": 136.555, "state": "S (sleeping)", "threads": 106}, "restart_count": 18, "restart_count_per_hour": 11, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 19}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 343.731, "sample_count": 777, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 78074, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:50:12+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.771, "last_frame_pushed_at": "2026-07-02T11:50:13+09:00", "last_frame_read_age_sec": 5.05, "last_frame_read_at": "2026-07-02T11:50:12+09:00", "mjpeg": {"age_sec": 4.753, "exists": true, "mtime": "2026-07-02T11:50:13.162757+09:00", "mtime_epoch": 1782960613.1627574, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199567}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:50:17+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1366.539, "name": "celery", "pid": 237, "rss_mb": 1316.672, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 87, "pushed_frames": 87, "read_frames": 86, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.922, "pushed": 2.922, "read": 2.888}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 12.317, "count": 87, "max_ms": 17.97, "p95_ms": 14.447}, "gstreamer_push": {"avg_ms": 0.874, "count": 87, "max_ms": 0.973, "p95_ms": 0.933}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.439, "count": 87, "max_ms": 77.602, "p95_ms": 69.756}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.406, "count": 87, "max_ms": 107.796, "p95_ms": 88.278}, "yolo11_640_fire_11class": {"avg_ms": 48.311, "count": 87, "max_ms": 68.266, "p95_ms": 62.386}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 66.293, "count": 87, "max_ms": 111.503, "p95_ms": 99.602}}, "truncated_count": 0}, "processing_total": {"avg_ms": 281.103, "count": 87, "max_ms": 330.147, "p95_ms": 313.371}, "read_interval": {"avg_ms": 290.66, "count": 86, "max_ms": 339.483, "p95_ms": 322.965}, "triton_wait": {"avg_ms": 70.15, "count": 87, "max_ms": 111.515, "p95_ms": 101.375}, "visualize": {"avg_ms": 12.575, "count": 87, "max_ms": 17.117, "p95_ms": 13.778}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.774, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 654}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4949.703, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56386aa21700] error while decoding MB 26 24, bytestream -11", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 15, "h264_decode_error_count": 24, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:52:16+09:00", "last_error_summary": "[h264 @ 0x56386aa21700] error while decoding MB 26 24, bytestream -11", "last_exit_at": "2026-07-02T11:50:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 120.6, "last_started_at": "2026-07-02T11:50:15+09:00", "pid": 5445, "process": {"exists": false, "pid": 5445}, "restart_count": 18, "restart_count_per_hour": 11, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 19}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 343.731, "sample_count": 931, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 78493, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:52:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.049, "last_frame_pushed_at": "2026-07-02T11:52:16+09:00", "last_frame_read_age_sec": 0.041, "last_frame_read_at": "2026-07-02T11:52:16+09:00", "mjpeg": {"age_sec": 0.03, "exists": true, "mtime": "2026-07-02T11:52:16.484779+09:00", "mtime_epoch": 1782960736.4847786, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200336}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:52:16+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1366.539, "name": "celery", "pid": 237, "rss_mb": 1316.926, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 94, "pushed_frames": 94, "read_frames": 94, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.146, "pushed": 3.146, "read": 3.146}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.568, "count": 93, "max_ms": 19.196, "p95_ms": 17.129}, "gstreamer_push": {"avg_ms": 0.882, "count": 94, "max_ms": 1.394, "p95_ms": 0.957}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.755, "count": 93, "max_ms": 82.483, "p95_ms": 66.986}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.276, "count": 93, "max_ms": 106.935, "p95_ms": 96.383}, "yolo11_640_fire_11class": {"avg_ms": 48.155, "count": 93, "max_ms": 71.946, "p95_ms": 59.305}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.206, "count": 93, "max_ms": 109.297, "p95_ms": 93.93}}, "truncated_count": 0}, "processing_total": {"avg_ms": 311.412, "count": 94, "max_ms": 362.275, "p95_ms": 347.987}, "read_interval": {"avg_ms": 320.877, "count": 94, "max_ms": 371.532, "p95_ms": 356.839}, "triton_wait": {"avg_ms": 72.243, "count": 93, "max_ms": 109.307, "p95_ms": 97.689}, "visualize": {"avg_ms": 14.068, "count": 93, "max_ms": 21.542, "p95_ms": 16.904}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.88, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 654}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4952.082, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T11:52:16+09:00", "last_error_summary": "[h264 @ 0x56386aa21700] error while decoding MB 26 24, bytestream -11", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 15, "h264_decode_error_count": 24, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:52:16+09:00", "last_error_summary": "[h264 @ 0x56386aa21700] error while decoding MB 26 24, bytestream -11", "last_exit_at": "2026-07-02T11:52:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 120.876, "last_started_at": "2026-07-02T11:50:15+09:00", "pid": 5445, "process": {"exists": false, "pid": 5445}, "restart_count": 18, "restart_count_per_hour": 11, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 19}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 343.731, "sample_count": 931, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 78494, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:52:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T11:52:16+09:00", "last_frame_read_age_sec": 0.317, "last_frame_read_at": "2026-07-02T11:52:16+09:00", "mjpeg": {"age_sec": 0.306, "exists": true, "mtime": "2026-07-02T11:52:16.484779+09:00", "mtime_epoch": 1782960736.4847786, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200336}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:52:16+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1366.539, "name": "celery", "pid": 237, "rss_mb": 1316.926, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 94, "pushed_frames": 94, "read_frames": 93, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.154, "pushed": 3.154, "read": 3.121}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.556, "count": 93, "max_ms": 19.196, "p95_ms": 17.129}, "gstreamer_push": {"avg_ms": 0.881, "count": 94, "max_ms": 1.394, "p95_ms": 0.957}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.571, "count": 93, "max_ms": 82.483, "p95_ms": 66.527}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.191, "count": 93, "max_ms": 106.935, "p95_ms": 96.383}, "yolo11_640_fire_11class": {"avg_ms": 48.053, "count": 93, "max_ms": 71.946, "p95_ms": 58.984}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.017, "count": 93, "max_ms": 109.297, "p95_ms": 93.93}}, "truncated_count": 0}, "processing_total": {"avg_ms": 311.284, "count": 94, "max_ms": 362.275, "p95_ms": 347.987}, "read_interval": {"avg_ms": 320.76, "count": 93, "max_ms": 371.532, "p95_ms": 356.839}, "triton_wait": {"avg_ms": 72.158, "count": 93, "max_ms": 109.307, "p95_ms": 97.689}, "visualize": {"avg_ms": 14.053, "count": 93, "max_ms": 21.542, "p95_ms": 16.904}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.799, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 654}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4952.199, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 15, "h264_decode_error_count": 24, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:52:16+09:00", "last_error_summary": "[h264 @ 0x56386aa21700] error while decoding MB 26 24, bytestream -11", "last_exit_at": "2026-07-02T11:52:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 120.879, "last_started_at": "2026-07-02T11:50:15+09:00", "pid": 5445, "process": {"exists": false, "pid": 5445}, "restart_count": 18, "restart_count_per_hour": 11, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 19}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 343.731, "sample_count": 931, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 78494, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:52:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T11:52:16+09:00", "last_frame_read_age_sec": 0.32, "last_frame_read_at": "2026-07-02T11:52:16+09:00", "mjpeg": {"age_sec": 0.309, "exists": true, "mtime": "2026-07-02T11:52:16.484779+09:00", "mtime_epoch": 1782960736.4847786, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200336}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:52:16+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1366.539, "name": "celery", "pid": 237, "rss_mb": 1316.926, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 94, "pushed_frames": 94, "read_frames": 93, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.154, "pushed": 3.154, "read": 3.121}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.556, "count": 93, "max_ms": 19.196, "p95_ms": 17.129}, "gstreamer_push": {"avg_ms": 0.881, "count": 94, "max_ms": 1.394, "p95_ms": 0.957}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.571, "count": 93, "max_ms": 82.483, "p95_ms": 66.527}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.191, "count": 93, "max_ms": 106.935, "p95_ms": 96.383}, "yolo11_640_fire_11class": {"avg_ms": 48.053, "count": 93, "max_ms": 71.946, "p95_ms": 58.984}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.017, "count": 93, "max_ms": 109.297, "p95_ms": 93.93}}, "truncated_count": 0}, "processing_total": {"avg_ms": 311.284, "count": 94, "max_ms": 362.275, "p95_ms": 347.987}, "read_interval": {"avg_ms": 320.76, "count": 93, "max_ms": 371.532, "p95_ms": 356.839}, "triton_wait": {"avg_ms": 72.158, "count": 93, "max_ms": 109.307, "p95_ms": 97.689}, "visualize": {"avg_ms": 14.053, "count": 93, "max_ms": 21.542, "p95_ms": 16.904}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.802, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 7, "tracking_summary": 654}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5067.785, "limit_mb": 8796093022207.996}, "detail": {"pid": 5552, "restart_count": 19, "start_count": 20}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 15, "h264_decode_error_count": 24, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:52:16+09:00", "last_error_summary": "[h264 @ 0x56386aa21700] error while decoding MB 26 24, bytestream -11", "last_exit_at": "2026-07-02T11:52:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T11:52:19+09:00", "pid": 5552, "process": {"exists": true, "fd_count": 5, "hwm_mb": 149.387, "name": "ffmpeg", "pid": 5552, "rss_mb": 149.387, "state": "S (sleeping)", "threads": 106}, "restart_count": 19, "restart_count_per_hour": 12, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 20}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 343.731, "sample_count": 920, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 78494, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:52:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.549, "last_frame_pushed_at": "2026-07-02T11:52:16+09:00", "last_frame_read_age_sec": 4.862, "last_frame_read_at": "2026-07-02T11:52:16+09:00", "mjpeg": {"age_sec": 4.531, "exists": true, "mtime": "2026-07-02T11:52:16.804784+09:00", "mtime_epoch": 1782960736.8047838, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199549}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:52:21+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1366.539, "name": "celery", "pid": 237, "rss_mb": 1316.926, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 80, "pushed_frames": 80, "read_frames": 79, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.68, "pushed": 2.68, "read": 2.647}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 14.628, "count": 79, "max_ms": 19.196, "p95_ms": 17.578}, "gstreamer_push": {"avg_ms": 0.881, "count": 80, "max_ms": 1.394, "p95_ms": 0.957}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.681, "count": 79, "max_ms": 82.483, "p95_ms": 68.494}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.114, "count": 79, "max_ms": 106.935, "p95_ms": 94.025}, "yolo11_640_fire_11class": {"avg_ms": 48.09, "count": 79, "max_ms": 71.946, "p95_ms": 64.585}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 70.419, "count": 79, "max_ms": 109.297, "p95_ms": 95.774}}, "truncated_count": 0}, "processing_total": {"avg_ms": 310.906, "count": 80, "max_ms": 362.275, "p95_ms": 343.787}, "read_interval": {"avg_ms": 320.358, "count": 79, "max_ms": 371.532, "p95_ms": 358.119}, "triton_wait": {"avg_ms": 72.217, "count": 79, "max_ms": 109.307, "p95_ms": 99.303}, "visualize": {"avg_ms": 13.998, "count": 79, "max_ms": 18.72, "p95_ms": 16.904}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.845, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 3, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 14, "tracking_summary": 656}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5150.777, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T11:52:16+09:00", "last_error_summary": "[h264 @ 0x56386aa21700] error while decoding MB 26 24, bytestream -11", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 15, "h264_decode_error_count": 24, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:52:16+09:00", "last_error_summary": "[h264 @ 0x56386aa21700] error while decoding MB 26 24, bytestream -11", "last_exit_at": "2026-07-02T11:54:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 139.296, "last_started_at": "2026-07-02T11:52:19+09:00", "pid": 5552, "process": {"exists": false, "pid": 5552}, "restart_count": 19, "restart_count_per_hour": 12, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 20}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 532.919, "sample_count": 814, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 78767, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:54:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T11:54:38+09:00", "last_frame_read_age_sec": 0.386, "last_frame_read_at": "2026-07-02T11:54:38+09:00", "mjpeg": {"age_sec": 0.376, "exists": true, "mtime": "2026-07-02T11:54:38.255184+09:00", "mtime_epoch": 1782960878.255184, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198966}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:54:38+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1395.82, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 56, "pushed_frames": 56, "read_frames": 55, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.895, "pushed": 1.895, "read": 1.862}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.745, "count": 56, "max_ms": 40.293, "p95_ms": 31.37}, "gstreamer_push": {"avg_ms": 0.914, "count": 56, "max_ms": 2.405, "p95_ms": 0.992}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.202, "count": 56, "max_ms": 87.454, "p95_ms": 71.17}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.736, "count": 56, "max_ms": 121.826, "p95_ms": 109.965}, "yolo11_640_fire_11class": {"avg_ms": 48.325, "count": 56, "max_ms": 73.705, "p95_ms": 67.934}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.955, "count": 56, "max_ms": 124.695, "p95_ms": 111.734}}, "truncated_count": 0}, "processing_total": {"avg_ms": 526.714, "count": 56, "max_ms": 899.327, "p95_ms": 781.713}, "read_interval": {"avg_ms": 539.127, "count": 55, "max_ms": 909.7, "p95_ms": 791.394}, "triton_wait": {"avg_ms": 82.728, "count": 56, "max_ms": 124.71, "p95_ms": 115.73}, "visualize": {"avg_ms": 25.142, "count": 56, "max_ms": 41.342, "p95_ms": 34.338}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.544, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 3, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 14, "tracking_summary": 656}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5153.27, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 15, "h264_decode_error_count": 24, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:52:16+09:00", "last_error_summary": "[h264 @ 0x56386aa21700] error while decoding MB 26 24, bytestream -11", "last_exit_at": "2026-07-02T11:54:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 139.299, "last_started_at": "2026-07-02T11:52:19+09:00", "pid": 5552, "process": {"exists": false, "pid": 5552}, "restart_count": 19, "restart_count_per_hour": 12, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 20}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 532.919, "sample_count": 814, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 78767, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:54:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T11:54:38+09:00", "last_frame_read_age_sec": 0.389, "last_frame_read_at": "2026-07-02T11:54:38+09:00", "mjpeg": {"age_sec": 0.379, "exists": true, "mtime": "2026-07-02T11:54:38.255184+09:00", "mtime_epoch": 1782960878.255184, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198966}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:54:38+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1395.82, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 56, "pushed_frames": 56, "read_frames": 55, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.895, "pushed": 1.895, "read": 1.861}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.745, "count": 56, "max_ms": 40.293, "p95_ms": 31.37}, "gstreamer_push": {"avg_ms": 0.914, "count": 56, "max_ms": 2.405, "p95_ms": 0.992}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.202, "count": 56, "max_ms": 87.454, "p95_ms": 71.17}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.736, "count": 56, "max_ms": 121.826, "p95_ms": 109.965}, "yolo11_640_fire_11class": {"avg_ms": 48.325, "count": 56, "max_ms": 73.705, "p95_ms": 67.934}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.955, "count": 56, "max_ms": 124.695, "p95_ms": 111.734}}, "truncated_count": 0}, "processing_total": {"avg_ms": 526.714, "count": 56, "max_ms": 899.327, "p95_ms": 781.713}, "read_interval": {"avg_ms": 539.127, "count": 55, "max_ms": 909.7, "p95_ms": 791.394}, "triton_wait": {"avg_ms": 82.728, "count": 56, "max_ms": 124.71, "p95_ms": 115.73}, "visualize": {"avg_ms": 25.142, "count": 56, "max_ms": 41.342, "p95_ms": 34.338}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.547, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 3, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 14, "tracking_summary": 656}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5258.434, "limit_mb": 8796093022207.996}, "detail": {"pid": 5840, "restart_count": 20, "start_count": 21}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 15, "h264_decode_error_count": 24, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:52:16+09:00", "last_error_summary": "[h264 @ 0x56386aa21700] error while decoding MB 26 24, bytestream -11", "last_exit_at": "2026-07-02T11:54:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T11:54:41+09:00", "pid": 5840, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.102, "name": "ffmpeg", "pid": 5840, "rss_mb": 139.102, "state": "S (sleeping)", "threads": 106}, "restart_count": 20, "restart_count_per_hour": 13, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 21}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 533.659, "sample_count": 795, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 78767, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:54:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.991, "last_frame_pushed_at": "2026-07-02T11:54:38+09:00", "last_frame_read_age_sec": 5.374, "last_frame_read_at": "2026-07-02T11:54:38+09:00", "mjpeg": {"age_sec": 4.972, "exists": true, "mtime": "2026-07-02T11:54:38.647191+09:00", "mtime_epoch": 1782960878.6471906, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199926}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:54:43+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1395.82, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 46, "pushed_frames": 46, "read_frames": 45, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.558, "pushed": 1.558, "read": 1.524}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.907, "count": 46, "max_ms": 40.293, "p95_ms": 31.37}, "gstreamer_push": {"avg_ms": 0.909, "count": 46, "max_ms": 2.405, "p95_ms": 0.95}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.997, "count": 46, "max_ms": 87.454, "p95_ms": 71.17}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.556, "count": 46, "max_ms": 121.826, "p95_ms": 109.965}, "yolo11_640_fire_11class": {"avg_ms": 48.492, "count": 46, "max_ms": 73.705, "p95_ms": 67.934}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.961, "count": 46, "max_ms": 124.695, "p95_ms": 111.734}}, "truncated_count": 0}, "processing_total": {"avg_ms": 534.198, "count": 46, "max_ms": 899.327, "p95_ms": 781.713}, "read_interval": {"avg_ms": 547.445, "count": 45, "max_ms": 909.7, "p95_ms": 791.394}, "triton_wait": {"avg_ms": 83.226, "count": 46, "max_ms": 124.71, "p95_ms": 115.73}, "visualize": {"avg_ms": 25.741, "count": 46, "max_ms": 41.342, "p95_ms": 34.338}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.521, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 16, "tracking_summary": 658}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4991.164, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5629bd414300] error while decoding MB 11 36, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 16, "h264_decode_error_count": 25, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:56:53+09:00", "last_error_summary": "[h264 @ 0x5629bd414300] error while decoding MB 11 36, bytestream -7", "last_exit_at": "2026-07-02T11:54:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.441, "last_started_at": "2026-07-02T11:54:41+09:00", "pid": 5840, "process": {"exists": false, "pid": 5840}, "restart_count": 20, "restart_count_per_hour": 13, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 21}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 559.558, "sample_count": 636, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 79055, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:56:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-02T11:56:52+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-02T11:56:53+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-02T11:56:53.013533+09:00", "mtime_epoch": 1782961013.0135326, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 202302}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:56:53+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1319.48, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 71, "pushed_frames": 71, "read_frames": 71, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.392, "pushed": 2.392, "read": 2.392}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 22.061, "count": 71, "max_ms": 29.898, "p95_ms": 26.202}, "gstreamer_push": {"avg_ms": 0.878, "count": 71, "max_ms": 1.329, "p95_ms": 0.938}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.094, "count": 71, "max_ms": 85.184, "p95_ms": 67.753}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.575, "count": 71, "max_ms": 110.413, "p95_ms": 100.445}, "yolo11_640_fire_11class": {"avg_ms": 48.905, "count": 71, "max_ms": 76.925, "p95_ms": 62.903}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.142, "count": 71, "max_ms": 123.63, "p95_ms": 108.094}}, "truncated_count": 0}, "processing_total": {"avg_ms": 414.202, "count": 71, "max_ms": 484.949, "p95_ms": 477.799}, "read_interval": {"avg_ms": 423.571, "count": 71, "max_ms": 495.229, "p95_ms": 485.928}, "triton_wait": {"avg_ms": 80.737, "count": 71, "max_ms": 123.642, "p95_ms": 108.106}, "visualize": {"avg_ms": 17.768, "count": 71, "max_ms": 20.516, "p95_ms": 19.837}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.68, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 16, "tracking_summary": 658}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4990.367, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T11:56:53+09:00", "last_error_summary": "[h264 @ 0x5629bd414300] error while decoding MB 11 36, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 16, "h264_decode_error_count": 25, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:56:53+09:00", "last_error_summary": "[h264 @ 0x5629bd414300] error while decoding MB 11 36, bytestream -7", "last_exit_at": "2026-07-02T11:56:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.776, "last_started_at": "2026-07-02T11:54:41+09:00", "pid": 5840, "process": {"exists": false, "pid": 5840}, "restart_count": 20, "restart_count_per_hour": 13, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 21}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 559.558, "sample_count": 636, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 79056, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:56:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.002, "last_frame_pushed_at": "2026-07-02T11:56:53+09:00", "last_frame_read_age_sec": 0.388, "last_frame_read_at": "2026-07-02T11:56:53+09:00", "mjpeg": {"age_sec": 0.381, "exists": true, "mtime": "2026-07-02T11:56:53.013533+09:00", "mtime_epoch": 1782961013.0135326, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 202302}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:56:53+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1319.48, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 71, "pushed_frames": 71, "read_frames": 70, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.403, "pushed": 2.403, "read": 2.369}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 21.97, "count": 71, "max_ms": 29.898, "p95_ms": 26.18}, "gstreamer_push": {"avg_ms": 0.877, "count": 71, "max_ms": 1.329, "p95_ms": 0.938}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.147, "count": 71, "max_ms": 85.184, "p95_ms": 67.753}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.594, "count": 71, "max_ms": 110.413, "p95_ms": 100.445}, "yolo11_640_fire_11class": {"avg_ms": 48.994, "count": 71, "max_ms": 76.925, "p95_ms": 62.903}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.142, "count": 71, "max_ms": 123.63, "p95_ms": 108.094}}, "truncated_count": 0}, "processing_total": {"avg_ms": 413.327, "count": 71, "max_ms": 484.949, "p95_ms": 477.799}, "read_interval": {"avg_ms": 423.088, "count": 70, "max_ms": 495.229, "p95_ms": 485.928}, "triton_wait": {"avg_ms": 80.756, "count": 71, "max_ms": 123.642, "p95_ms": 108.106}, "visualize": {"avg_ms": 17.727, "count": 71, "max_ms": 20.516, "p95_ms": 19.837}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.55, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 16, "tracking_summary": 658}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4990.359, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 16, "h264_decode_error_count": 25, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:56:53+09:00", "last_error_summary": "[h264 @ 0x5629bd414300] error while decoding MB 11 36, bytestream -7", "last_exit_at": "2026-07-02T11:56:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.779, "last_started_at": "2026-07-02T11:54:41+09:00", "pid": 5840, "process": {"exists": false, "pid": 5840}, "restart_count": 20, "restart_count_per_hour": 13, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 21}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 559.558, "sample_count": 636, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 79056, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:56:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T11:56:53+09:00", "last_frame_read_age_sec": 0.391, "last_frame_read_at": "2026-07-02T11:56:53+09:00", "mjpeg": {"age_sec": 0.384, "exists": true, "mtime": "2026-07-02T11:56:53.013533+09:00", "mtime_epoch": 1782961013.0135326, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 202302}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:56:53+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1319.48, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 71, "pushed_frames": 71, "read_frames": 70, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.402, "pushed": 2.402, "read": 2.369}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 21.97, "count": 71, "max_ms": 29.898, "p95_ms": 26.18}, "gstreamer_push": {"avg_ms": 0.877, "count": 71, "max_ms": 1.329, "p95_ms": 0.938}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.147, "count": 71, "max_ms": 85.184, "p95_ms": 67.753}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 78.594, "count": 71, "max_ms": 110.413, "p95_ms": 100.445}, "yolo11_640_fire_11class": {"avg_ms": 48.994, "count": 71, "max_ms": 76.925, "p95_ms": 62.903}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 78.142, "count": 71, "max_ms": 123.63, "p95_ms": 108.094}}, "truncated_count": 0}, "processing_total": {"avg_ms": 413.327, "count": 71, "max_ms": 484.949, "p95_ms": 477.799}, "read_interval": {"avg_ms": 423.088, "count": 70, "max_ms": 495.229, "p95_ms": 485.928}, "triton_wait": {"avg_ms": 80.756, "count": 71, "max_ms": 123.642, "p95_ms": 108.106}, "visualize": {"avg_ms": 17.727, "count": 71, "max_ms": 20.516, "p95_ms": 19.837}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.553, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 16, "tracking_summary": 658}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5098.102, "limit_mb": 8796093022207.996}, "detail": {"pid": 6007, "restart_count": 21, "start_count": 22}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 16, "h264_decode_error_count": 25, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:56:53+09:00", "last_error_summary": "[h264 @ 0x5629bd414300] error while decoding MB 11 36, bytestream -7", "last_exit_at": "2026-07-02T11:56:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T11:56:55+09:00", "pid": 6007, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.84, "name": "ffmpeg", "pid": 6007, "rss_mb": 136.84, "state": "S (sleeping)", "threads": 106}, "restart_count": 21, "restart_count_per_hour": 14, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 22}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 560.166, "sample_count": 623, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 79056, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:56:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.2, "last_frame_pushed_at": "2026-07-02T11:56:53+09:00", "last_frame_read_age_sec": 4.586, "last_frame_read_at": "2026-07-02T11:56:53+09:00", "mjpeg": {"age_sec": 4.183, "exists": true, "mtime": "2026-07-02T11:56:53.409540+09:00", "mtime_epoch": 1782961013.4095397, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 202452}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:56:57+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1319.48, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 62, "pushed_frames": 62, "read_frames": 61, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.095, "pushed": 2.095, "read": 2.061}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 21.85, "count": 62, "max_ms": 29.898, "p95_ms": 25.796}, "gstreamer_push": {"avg_ms": 0.878, "count": 62, "max_ms": 1.329, "p95_ms": 0.928}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.927, "count": 62, "max_ms": 85.184, "p95_ms": 66.378}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 77.416, "count": 62, "max_ms": 110.413, "p95_ms": 100.445}, "yolo11_640_fire_11class": {"avg_ms": 48.821, "count": 62, "max_ms": 76.925, "p95_ms": 60.253}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.214, "count": 62, "max_ms": 123.63, "p95_ms": 100.141}}, "truncated_count": 0}, "processing_total": {"avg_ms": 407.63, "count": 62, "max_ms": 484.949, "p95_ms": 470.748}, "read_interval": {"avg_ms": 417.304, "count": 61, "max_ms": 495.229, "p95_ms": 479.562}, "triton_wait": {"avg_ms": 79.368, "count": 62, "max_ms": 123.642, "p95_ms": 103.359}, "visualize": {"avg_ms": 17.534, "count": 62, "max_ms": 20.285, "p95_ms": 19.635}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.597, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 26, "tracking_summary": 636}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4930.0, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[NULL @ 0x55de4ef96200] missing picture in access unit with size 18 [h264 @ 0x55de4f7024c0] no frame! [vist#0:0/h264 @ 0x55de4ef96840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 13, "h264_decode_error_count": 13, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:57:23+09:00", "last_error_summary": "[NULL @ 0x55de4ef96200] missing picture in access unit with size 18 [h264 @ 0x55de4f7024c0] no frame! [vist#0:0/h264 @ 0x55de4ef96840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T11:48:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 522.402, "last_started_at": "2026-07-02T11:48:41+09:00", "pid": 5233, "process": {"exists": false, "pid": 5233}, "restart_count": 12, "restart_count_per_hour": 7, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 13}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 245.235, "sample_count": 1556, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 76397, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:57:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.083, "last_frame_pushed_at": "2026-07-02T11:57:23+09:00", "last_frame_read_age_sec": 0.074, "last_frame_read_at": "2026-07-02T11:57:23+09:00", "mjpeg": {"age_sec": 0.063, "exists": true, "mtime": "2026-07-02T11:57:23.758075+09:00", "mtime_epoch": 1782961043.7580755, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 259282}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T11:57:23+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1067.082, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 146, "pushed_frames": 146, "read_frames": 146, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.884, "pushed": 4.884, "read": 4.884}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.652, "count": 146, "max_ms": 14.367, "p95_ms": 12.582}, "gstreamer_push": {"avg_ms": 0.872, "count": 146, "max_ms": 1.375, "p95_ms": 0.916}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.242, "count": 146, "max_ms": 77.9, "p95_ms": 61.784}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.044, "count": 146, "max_ms": 122.401, "p95_ms": 95.459}, "yolo11_640_fire_11class": {"avg_ms": 48.347, "count": 146, "max_ms": 72.835, "p95_ms": 57.465}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.933, "count": 146, "max_ms": 119.932, "p95_ms": 100.021}}, "truncated_count": 0}, "processing_total": {"avg_ms": 195.459, "count": 146, "max_ms": 280.474, "p95_ms": 238.081}, "read_interval": {"avg_ms": 205.502, "count": 146, "max_ms": 290.986, "p95_ms": 247.473}, "triton_wait": {"avg_ms": 66.502, "count": 146, "max_ms": 122.413, "p95_ms": 100.839}, "visualize": {"avg_ms": 12.084, "count": 146, "max_ms": 15.696, "p95_ms": 13.239}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.894, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 26, "tracking_summary": 636}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4929.445, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T11:57:23+09:00", "last_error_summary": "[NULL @ 0x55de4ef96200] missing picture in access unit with size 18 [h264 @ 0x55de4f7024c0] no frame! [vist#0:0/h264 @ 0x55de4ef96840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 13, "h264_decode_error_count": 13, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:57:23+09:00", "last_error_summary": "[NULL @ 0x55de4ef96200] missing picture in access unit with size 18 [h264 @ 0x55de4f7024c0] no frame! [vist#0:0/h264 @ 0x55de4ef96840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T11:57:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 522.559, "last_started_at": "2026-07-02T11:48:41+09:00", "pid": 5233, "process": {"exists": false, "pid": 5233}, "restart_count": 12, "restart_count_per_hour": 7, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 13}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 245.235, "sample_count": 1556, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 76398, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:57:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T11:57:23+09:00", "last_frame_read_age_sec": 0.231, "last_frame_read_at": "2026-07-02T11:57:23+09:00", "mjpeg": {"age_sec": 0.22, "exists": true, "mtime": "2026-07-02T11:57:23.758075+09:00", "mtime_epoch": 1782961043.7580755, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 259282}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T11:57:23+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1067.082, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 146, "pushed_frames": 146, "read_frames": 145, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.89, "pushed": 4.89, "read": 4.857}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.667, "count": 146, "max_ms": 14.367, "p95_ms": 12.582}, "gstreamer_push": {"avg_ms": 0.872, "count": 146, "max_ms": 1.375, "p95_ms": 0.916}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.276, "count": 146, "max_ms": 77.9, "p95_ms": 61.784}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.06, "count": 146, "max_ms": 122.401, "p95_ms": 95.459}, "yolo11_640_fire_11class": {"avg_ms": 48.346, "count": 146, "max_ms": 72.835, "p95_ms": 57.465}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.952, "count": 146, "max_ms": 119.932, "p95_ms": 100.021}}, "truncated_count": 0}, "processing_total": {"avg_ms": 195.75, "count": 146, "max_ms": 280.474, "p95_ms": 238.081}, "read_interval": {"avg_ms": 205.583, "count": 145, "max_ms": 290.986, "p95_ms": 247.473}, "triton_wait": {"avg_ms": 66.522, "count": 146, "max_ms": 122.413, "p95_ms": 100.839}, "visualize": {"avg_ms": 12.129, "count": 146, "max_ms": 18.838, "p95_ms": 13.366}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.857, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 26, "tracking_summary": 636}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4929.438, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 13, "h264_decode_error_count": 13, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:57:23+09:00", "last_error_summary": "[NULL @ 0x55de4ef96200] missing picture in access unit with size 18 [h264 @ 0x55de4f7024c0] no frame! [vist#0:0/h264 @ 0x55de4ef96840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T11:57:23+09:00", "last_returncode": 0, "last_stable_duration_sec": 522.563, "last_started_at": "2026-07-02T11:48:41+09:00", "pid": 5233, "process": {"exists": false, "pid": 5233}, "restart_count": 12, "restart_count_per_hour": 7, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 13}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 245.235, "sample_count": 1556, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 76398, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:57:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T11:57:23+09:00", "last_frame_read_age_sec": 0.235, "last_frame_read_at": "2026-07-02T11:57:23+09:00", "mjpeg": {"age_sec": 0.224, "exists": true, "mtime": "2026-07-02T11:57:23.758075+09:00", "mtime_epoch": 1782961043.7580755, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 259282}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T11:57:23+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1067.082, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 146, "pushed_frames": 146, "read_frames": 145, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.889, "pushed": 4.889, "read": 4.856}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.667, "count": 146, "max_ms": 14.367, "p95_ms": 12.582}, "gstreamer_push": {"avg_ms": 0.872, "count": 146, "max_ms": 1.375, "p95_ms": 0.916}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.276, "count": 146, "max_ms": 77.9, "p95_ms": 61.784}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.06, "count": 146, "max_ms": 122.401, "p95_ms": 95.459}, "yolo11_640_fire_11class": {"avg_ms": 48.346, "count": 146, "max_ms": 72.835, "p95_ms": 57.465}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.952, "count": 146, "max_ms": 119.932, "p95_ms": 100.021}}, "truncated_count": 0}, "processing_total": {"avg_ms": 195.75, "count": 146, "max_ms": 280.474, "p95_ms": 238.081}, "read_interval": {"avg_ms": 205.583, "count": 145, "max_ms": 290.986, "p95_ms": 247.473}, "triton_wait": {"avg_ms": 66.522, "count": 146, "max_ms": 122.413, "p95_ms": 100.839}, "visualize": {"avg_ms": 12.129, "count": 146, "max_ms": 18.838, "p95_ms": 13.366}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.86, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 26, "tracking_summary": 636}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 4949.867, "limit_mb": 8796093022207.996}, "detail": {"pid": 6114, "restart_count": 13, "start_count": 14}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 13, "h264_decode_error_count": 13, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:57:23+09:00", "last_error_summary": "[NULL @ 0x55de4ef96200] missing picture in access unit with size 18 [h264 @ 0x55de4f7024c0] no frame! [vist#0:0/h264 @ 0x55de4ef96840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T11:57:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T11:57:26+09:00", "pid": 6114, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.676, "name": "ffmpeg", "pid": 6114, "rss_mb": 57.676, "state": "S (sleeping)", "threads": 1}, "restart_count": 13, "restart_count_per_hour": 8, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 14}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 245.5, "sample_count": 1524, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 76398, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:57:23+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 4.922, "last_frame_pushed_at": "2026-07-02T11:57:23+09:00", "last_frame_read_age_sec": 5.15, "last_frame_read_at": "2026-07-02T11:57:23+09:00", "mjpeg": {"age_sec": 4.903, "exists": true, "mtime": "2026-07-02T11:57:23.994080+09:00", "mtime_epoch": 1782961043.9940796, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 259259}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T11:57:28+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1067.082, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 121, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.067, "pushed": 4.067, "read": 4.034}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.775, "count": 121, "max_ms": 14.367, "p95_ms": 12.775}, "gstreamer_push": {"avg_ms": 0.871, "count": 122, "max_ms": 1.375, "p95_ms": 0.916}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.588, "count": 121, "max_ms": 77.9, "p95_ms": 62.306}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.445, "count": 121, "max_ms": 122.401, "p95_ms": 99.314}, "yolo11_640_fire_11class": {"avg_ms": 48.515, "count": 121, "max_ms": 72.835, "p95_ms": 57.985}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.805, "count": 121, "max_ms": 119.932, "p95_ms": 100.089}}, "truncated_count": 0}, "processing_total": {"avg_ms": 197.094, "count": 122, "max_ms": 280.474, "p95_ms": 238.36}, "read_interval": {"avg_ms": 206.896, "count": 121, "max_ms": 290.986, "p95_ms": 247.934}, "triton_wait": {"avg_ms": 67.273, "count": 121, "max_ms": 122.413, "p95_ms": 103.952}, "visualize": {"avg_ms": 12.142, "count": 121, "max_ms": 18.838, "p95_ms": 13.648}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 16, "tracking_summary": 661}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4988.91, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55bb73d99680] error while decoding MB 50 21, bytestream -24", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 17, "h264_decode_error_count": 26, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:59:08+09:00", "last_error_summary": "[h264 @ 0x55bb73d99680] error while decoding MB 50 21, bytestream -24", "last_exit_at": "2026-07-02T11:56:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.693, "last_started_at": "2026-07-02T11:56:55+09:00", "pid": 6007, "process": {"exists": false, "pid": 6007}, "restart_count": 21, "restart_count_per_hour": 14, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 22}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 550.9, "sample_count": 629, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 79339, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:59:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-02T11:59:08+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-02T11:59:08+09:00", "mjpeg": {"age_sec": 0.048, "exists": true, "mtime": "2026-07-02T11:59:08.236432+09:00", "mtime_epoch": 1782961148.2364318, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 219152}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:59:08+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1319.48, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 69, "pushed_frames": 69, "read_frames": 69, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.312, "pushed": 2.312, "read": 2.312}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 22.516, "count": 68, "max_ms": 25.39, "p95_ms": 24.793}, "gstreamer_push": {"avg_ms": 0.885, "count": 69, "max_ms": 1.545, "p95_ms": 0.976}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.756, "count": 68, "max_ms": 69.91, "p95_ms": 63.646}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.059, "count": 68, "max_ms": 109.044, "p95_ms": 105.839}, "yolo11_640_fire_11class": {"avg_ms": 49.927, "count": 68, "max_ms": 73.232, "p95_ms": 64.451}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.026, "count": 68, "max_ms": 108.572, "p95_ms": 102.377}}, "truncated_count": 0}, "processing_total": {"avg_ms": 428.355, "count": 69, "max_ms": 487.027, "p95_ms": 454.202}, "read_interval": {"avg_ms": 437.547, "count": 69, "max_ms": 496.054, "p95_ms": 464.097}, "triton_wait": {"avg_ms": 83.015, "count": 68, "max_ms": 109.057, "p95_ms": 105.852}, "visualize": {"avg_ms": 18.302, "count": 68, "max_ms": 21.992, "p95_ms": 19.532}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.847, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 16, "tracking_summary": 661}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4988.371, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T11:59:08+09:00", "last_error_summary": "[h264 @ 0x55bb73d99680] error while decoding MB 50 21, bytestream -24", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 17, "h264_decode_error_count": 26, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:59:08+09:00", "last_error_summary": "[h264 @ 0x55bb73d99680] error while decoding MB 50 21, bytestream -24", "last_exit_at": "2026-07-02T11:59:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 133.059, "last_started_at": "2026-07-02T11:56:55+09:00", "pid": 6007, "process": {"exists": false, "pid": 6007}, "restart_count": 21, "restart_count_per_hour": 14, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 22}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 547.11, "sample_count": 629, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 79340, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:59:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T11:59:08+09:00", "last_frame_read_age_sec": 0.421, "last_frame_read_at": "2026-07-02T11:59:08+09:00", "mjpeg": {"age_sec": 0.413, "exists": true, "mtime": "2026-07-02T11:59:08.236432+09:00", "mtime_epoch": 1782961148.2364318, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 219152}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:59:08+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1319.48, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 69, "pushed_frames": 69, "read_frames": 68, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.316, "pushed": 2.316, "read": 2.283}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 22.492, "count": 68, "max_ms": 25.39, "p95_ms": 24.793}, "gstreamer_push": {"avg_ms": 0.886, "count": 69, "max_ms": 1.545, "p95_ms": 0.976}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.813, "count": 68, "max_ms": 69.91, "p95_ms": 63.646}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.182, "count": 68, "max_ms": 109.044, "p95_ms": 105.839}, "yolo11_640_fire_11class": {"avg_ms": 49.993, "count": 68, "max_ms": 73.232, "p95_ms": 64.451}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.112, "count": 68, "max_ms": 108.572, "p95_ms": 102.377}}, "truncated_count": 0}, "processing_total": {"avg_ms": 428.592, "count": 69, "max_ms": 487.027, "p95_ms": 454.202}, "read_interval": {"avg_ms": 437.962, "count": 68, "max_ms": 496.054, "p95_ms": 464.097}, "triton_wait": {"avg_ms": 83.101, "count": 68, "max_ms": 109.057, "p95_ms": 105.852}, "visualize": {"avg_ms": 18.299, "count": 68, "max_ms": 21.992, "p95_ms": 19.532}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.79, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 16, "tracking_summary": 661}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 4988.363, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 17, "h264_decode_error_count": 26, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:59:08+09:00", "last_error_summary": "[h264 @ 0x55bb73d99680] error while decoding MB 50 21, bytestream -24", "last_exit_at": "2026-07-02T11:59:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 133.062, "last_started_at": "2026-07-02T11:56:55+09:00", "pid": 6007, "process": {"exists": false, "pid": 6007}, "restart_count": 21, "restart_count_per_hour": 14, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 22}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 547.11, "sample_count": 629, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 79340, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:59:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T11:59:08+09:00", "last_frame_read_age_sec": 0.423, "last_frame_read_at": "2026-07-02T11:59:08+09:00", "mjpeg": {"age_sec": 0.416, "exists": true, "mtime": "2026-07-02T11:59:08.236432+09:00", "mtime_epoch": 1782961148.2364318, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 219152}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:59:08+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1319.48, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 69, "pushed_frames": 69, "read_frames": 68, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.316, "pushed": 2.316, "read": 2.282}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 22.492, "count": 68, "max_ms": 25.39, "p95_ms": 24.793}, "gstreamer_push": {"avg_ms": 0.886, "count": 69, "max_ms": 1.545, "p95_ms": 0.976}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.813, "count": 68, "max_ms": 69.91, "p95_ms": 63.646}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 80.182, "count": 68, "max_ms": 109.044, "p95_ms": 105.839}, "yolo11_640_fire_11class": {"avg_ms": 49.993, "count": 68, "max_ms": 73.232, "p95_ms": 64.451}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.112, "count": 68, "max_ms": 108.572, "p95_ms": 102.377}}, "truncated_count": 0}, "processing_total": {"avg_ms": 428.592, "count": 69, "max_ms": 487.027, "p95_ms": 454.202}, "read_interval": {"avg_ms": 437.962, "count": 68, "max_ms": 496.054, "p95_ms": 464.097}, "triton_wait": {"avg_ms": 83.101, "count": 68, "max_ms": 109.057, "p95_ms": 105.852}, "visualize": {"avg_ms": 18.299, "count": 68, "max_ms": 21.992, "p95_ms": 19.532}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.793, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 16, "tracking_summary": 661}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5105.262, "limit_mb": 8796093022207.996}, "detail": {"pid": 6266, "restart_count": 22, "start_count": 23}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 17, "h264_decode_error_count": 26, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:59:08+09:00", "last_error_summary": "[h264 @ 0x55bb73d99680] error while decoding MB 50 21, bytestream -24", "last_exit_at": "2026-07-02T11:59:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T11:59:10+09:00", "pid": 6266, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.0, "name": "ffmpeg", "pid": 6266, "rss_mb": 136.0, "state": "S (sleeping)", "threads": 106}, "restart_count": 22, "restart_count_per_hour": 15, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 23}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 547.11, "sample_count": 621, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 79340, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T11:59:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.11, "last_frame_pushed_at": "2026-07-02T11:59:08+09:00", "last_frame_read_age_sec": 4.527, "last_frame_read_at": "2026-07-02T11:59:08+09:00", "mjpeg": {"age_sec": 4.095, "exists": true, "mtime": "2026-07-02T11:59:08.660444+09:00", "mtime_epoch": 1782961148.660444, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 213255}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T11:59:12+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1319.48, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 60, "pushed_frames": 60, "read_frames": 59, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.006, "pushed": 2.006, "read": 1.972}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 22.428, "count": 59, "max_ms": 25.39, "p95_ms": 24.793}, "gstreamer_push": {"avg_ms": 0.878, "count": 60, "max_ms": 1.214, "p95_ms": 0.97}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.216, "count": 59, "max_ms": 65.751, "p95_ms": 63.012}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 79.589, "count": 59, "max_ms": 105.955, "p95_ms": 104.944}, "yolo11_640_fire_11class": {"avg_ms": 49.442, "count": 59, "max_ms": 65.101, "p95_ms": 64.451}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.851, "count": 59, "max_ms": 103.646, "p95_ms": 101.068}}, "truncated_count": 0}, "processing_total": {"avg_ms": 428.067, "count": 60, "max_ms": 487.027, "p95_ms": 452.686}, "read_interval": {"avg_ms": 437.409, "count": 59, "max_ms": 496.054, "p95_ms": 464.097}, "triton_wait": {"avg_ms": 82.786, "count": 59, "max_ms": 105.968, "p95_ms": 104.956}, "visualize": {"avg_ms": 18.292, "count": 59, "max_ms": 21.992, "p95_ms": 20.28}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.916, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 18, "tracking_summary": 663}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5164.867, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T11:59:08+09:00", "last_error_summary": "[h264 @ 0x55bb73d99680] error while decoding MB 50 21, bytestream -24", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 17, "h264_decode_error_count": 26, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:59:08+09:00", "last_error_summary": "[h264 @ 0x55bb73d99680] error while decoding MB 50 21, bytestream -24", "last_exit_at": "2026-07-02T12:01:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 137.325, "last_started_at": "2026-07-02T11:59:10+09:00", "pid": 6266, "process": {"exists": false, "pid": 6266}, "restart_count": 22, "restart_count_per_hour": 15, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 23}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 489.897, "sample_count": 644, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 79639, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:01:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T12:01:28+09:00", "last_frame_read_age_sec": 0.496, "last_frame_read_at": "2026-07-02T12:01:27+09:00", "mjpeg": {"age_sec": 0.488, "exists": true, "mtime": "2026-07-02T12:01:27.592257+09:00", "mtime_epoch": 1782961287.5922575, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205850}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:01:28+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1319.738, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": 16.001, "recorders": [{"age_sec": 16.001, "frame_count": 81, "is_recording": true, "is_stopping": false, "pid": 6373, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.281, "name": "ffmpeg", "pid": 6373, "rss_mb": 186.281, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 19, "returncode": null, "running": true, "started_at": "2026-07-02T12:01:12+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_120111_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [6373], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 61, "pushed_frames": 61, "read_frames": 60, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.051, "pushed": 2.051, "read": 2.018}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 24.576, "count": 60, "max_ms": 31.301, "p95_ms": 29.065}, "gstreamer_push": {"avg_ms": 0.873, "count": 61, "max_ms": 1.02, "p95_ms": 0.945}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.465, "count": 60, "max_ms": 74.902, "p95_ms": 66.35}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 84.263, "count": 60, "max_ms": 126.581, "p95_ms": 113.229}, "yolo11_640_fire_11class": {"avg_ms": 49.415, "count": 60, "max_ms": 77.259, "p95_ms": 67.579}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 81.26, "count": 60, "max_ms": 115.062, "p95_ms": 111.071}}, "truncated_count": 0}, "processing_total": {"avg_ms": 484.814, "count": 61, "max_ms": 1201.738, "p95_ms": 529.788}, "read_interval": {"avg_ms": 494.319, "count": 60, "max_ms": 1211.776, "p95_ms": 541.628}, "triton_wait": {"avg_ms": 85.25, "count": 60, "max_ms": 126.594, "p95_ms": 113.241}, "visualize": {"avg_ms": 19.678, "count": 60, "max_ms": 24.873, "p95_ms": 23.52}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.737, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 18, "tracking_summary": 663}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5164.984, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 17, "h264_decode_error_count": 26, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:59:08+09:00", "last_error_summary": "[h264 @ 0x55bb73d99680] error while decoding MB 50 21, bytestream -24", "last_exit_at": "2026-07-02T12:01:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 137.328, "last_started_at": "2026-07-02T11:59:10+09:00", "pid": 6266, "process": {"exists": false, "pid": 6266}, "restart_count": 22, "restart_count_per_hour": 15, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 23}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 489.897, "sample_count": 644, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 79639, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:01:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T12:01:28+09:00", "last_frame_read_age_sec": 0.499, "last_frame_read_at": "2026-07-02T12:01:27+09:00", "mjpeg": {"age_sec": 0.491, "exists": true, "mtime": "2026-07-02T12:01:27.592257+09:00", "mtime_epoch": 1782961287.5922575, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205850}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:01:28+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1319.738, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": 16.004, "recorders": [{"age_sec": 16.004, "frame_count": 81, "is_recording": true, "is_stopping": false, "pid": 6373, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.281, "name": "ffmpeg", "pid": 6373, "rss_mb": 186.281, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 19, "returncode": null, "running": true, "started_at": "2026-07-02T12:01:12+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_120111_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [6373], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 61, "pushed_frames": 61, "read_frames": 60, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.051, "pushed": 2.051, "read": 2.018}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 24.576, "count": 60, "max_ms": 31.301, "p95_ms": 29.065}, "gstreamer_push": {"avg_ms": 0.873, "count": 61, "max_ms": 1.02, "p95_ms": 0.945}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.465, "count": 60, "max_ms": 74.902, "p95_ms": 66.35}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 84.263, "count": 60, "max_ms": 126.581, "p95_ms": 113.229}, "yolo11_640_fire_11class": {"avg_ms": 49.415, "count": 60, "max_ms": 77.259, "p95_ms": 67.579}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 81.26, "count": 60, "max_ms": 115.062, "p95_ms": 111.071}}, "truncated_count": 0}, "processing_total": {"avg_ms": 484.814, "count": 61, "max_ms": 1201.738, "p95_ms": 529.788}, "read_interval": {"avg_ms": 494.319, "count": 60, "max_ms": 1211.776, "p95_ms": 541.628}, "triton_wait": {"avg_ms": 85.25, "count": 60, "max_ms": 126.594, "p95_ms": 113.241}, "visualize": {"avg_ms": 19.678, "count": 60, "max_ms": 24.873, "p95_ms": 23.52}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.74, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 18, "tracking_summary": 663}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5273.949, "limit_mb": 8796093022207.996}, "detail": {"pid": 6433, "restart_count": 23, "start_count": 24}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 17, "h264_decode_error_count": 26, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T11:59:08+09:00", "last_error_summary": "[h264 @ 0x55bb73d99680] error while decoding MB 50 21, bytestream -24", "last_exit_at": "2026-07-02T12:01:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T12:01:30+09:00", "pid": 6433, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.262, "name": "ffmpeg", "pid": 6433, "rss_mb": 139.262, "state": "S (sleeping)", "threads": 106}, "restart_count": 23, "restart_count_per_hour": 16, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 24}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 489.897, "sample_count": 635, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 79639, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:01:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.109, "last_frame_pushed_at": "2026-07-02T12:01:28+09:00", "last_frame_read_age_sec": 4.602, "last_frame_read_at": "2026-07-02T12:01:27+09:00", "mjpeg": {"age_sec": 4.094, "exists": true, "mtime": "2026-07-02T12:01:28.092271+09:00", "mtime_epoch": 1782961288.0922709, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206107}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:01:32+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1319.738, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": 20.109, "recorders": [{"age_sec": 20.109, "frame_count": 81, "is_recording": true, "is_stopping": false, "pid": 6373, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.281, "name": "ffmpeg", "pid": 6373, "rss_mb": 186.281, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 19, "returncode": null, "running": true, "started_at": "2026-07-02T12:01:12+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_120111_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [6373], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 52, "pushed_frames": 52, "read_frames": 51, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.743, "pushed": 1.743, "read": 1.71}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 25.076, "count": 51, "max_ms": 31.301, "p95_ms": 30.291}, "gstreamer_push": {"avg_ms": 0.872, "count": 52, "max_ms": 1.02, "p95_ms": 0.948}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.817, "count": 51, "max_ms": 74.902, "p95_ms": 67.017}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 84.256, "count": 51, "max_ms": 126.581, "p95_ms": 115.575}, "yolo11_640_fire_11class": {"avg_ms": 49.06, "count": 51, "max_ms": 77.259, "p95_ms": 69.482}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.251, "count": 51, "max_ms": 115.062, "p95_ms": 111.071}}, "truncated_count": 0}, "processing_total": {"avg_ms": 493.12, "count": 52, "max_ms": 1201.738, "p95_ms": 533.306}, "read_interval": {"avg_ms": 502.844, "count": 51, "max_ms": 1211.776, "p95_ms": 543.648}, "triton_wait": {"avg_ms": 84.666, "count": 51, "max_ms": 126.594, "p95_ms": 115.587}, "visualize": {"avg_ms": 19.897, "count": 51, "max_ms": 24.873, "p95_ms": 23.571}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.829, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 22, "tracking_summary": 665}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5085.613, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55b2cdf1fe00] error while decoding MB 12 43, bytestream -28", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 18, "h264_decode_error_count": 27, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:03:53+09:00", "last_error_summary": "[h264 @ 0x55b2cdf1fe00] error while decoding MB 12 43, bytestream -28", "last_exit_at": "2026-07-02T12:01:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 143.736, "last_started_at": "2026-07-02T12:01:30+09:00", "pid": 6433, "process": {"exists": false, "pid": 6433}, "restart_count": 23, "restart_count_per_hour": 16, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 24}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 605.282, "sample_count": 580, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 79886, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:03:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-02T12:03:53+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-02T12:03:53+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-02T12:03:53.876027+09:00", "mtime_epoch": 1782961433.876027, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206292}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:03:53+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1319.91, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 55, "pushed_frames": 55, "read_frames": 55, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.856, "pushed": 1.856, "read": 1.856}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 28.489, "count": 54, "max_ms": 42.971, "p95_ms": 32.449}, "gstreamer_push": {"avg_ms": 0.87, "count": 55, "max_ms": 0.966, "p95_ms": 0.93}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.476, "count": 54, "max_ms": 73.6, "p95_ms": 64.446}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 89.806, "count": 54, "max_ms": 128.914, "p95_ms": 110.215}, "yolo11_640_fire_11class": {"avg_ms": 46.603, "count": 54, "max_ms": 70.447, "p95_ms": 60.162}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.652, "count": 54, "max_ms": 125.139, "p95_ms": 106.305}}, "truncated_count": 0}, "processing_total": {"avg_ms": 537.86, "count": 55, "max_ms": 637.994, "p95_ms": 574.471}, "read_interval": {"avg_ms": 547.643, "count": 55, "max_ms": 646.755, "p95_ms": 583.279}, "triton_wait": {"avg_ms": 90.859, "count": 54, "max_ms": 128.926, "p95_ms": 111.782}, "visualize": {"avg_ms": 21.009, "count": 54, "max_ms": 26.742, "p95_ms": 22.802}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.639, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 22, "tracking_summary": 665}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5084.996, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T12:03:53+09:00", "last_error_summary": "[h264 @ 0x55b2cdf1fe00] error while decoding MB 12 43, bytestream -28", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 18, "h264_decode_error_count": 27, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:03:53+09:00", "last_error_summary": "[h264 @ 0x55b2cdf1fe00] error while decoding MB 12 43, bytestream -28", "last_exit_at": "2026-07-02T12:03:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 144.241, "last_started_at": "2026-07-02T12:01:30+09:00", "pid": 6433, "process": {"exists": false, "pid": 6433}, "restart_count": 23, "restart_count_per_hour": 16, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 24}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 605.282, "sample_count": 580, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 79887, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:03:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T12:03:54+09:00", "last_frame_read_age_sec": 0.556, "last_frame_read_at": "2026-07-02T12:03:53+09:00", "mjpeg": {"age_sec": 0.55, "exists": true, "mtime": "2026-07-02T12:03:53.876027+09:00", "mtime_epoch": 1782961433.876027, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206292}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:03:54+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1319.91, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 55, "pushed_frames": 55, "read_frames": 54, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.858, "pushed": 1.858, "read": 1.824}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 28.487, "count": 55, "max_ms": 42.971, "p95_ms": 32.449}, "gstreamer_push": {"avg_ms": 0.87, "count": 55, "max_ms": 0.966, "p95_ms": 0.93}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.371, "count": 55, "max_ms": 73.6, "p95_ms": 64.446}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 90.101, "count": 55, "max_ms": 128.914, "p95_ms": 110.215}, "yolo11_640_fire_11class": {"avg_ms": 46.53, "count": 55, "max_ms": 70.447, "p95_ms": 60.162}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.695, "count": 55, "max_ms": 125.139, "p95_ms": 106.305}}, "truncated_count": 0}, "processing_total": {"avg_ms": 538.217, "count": 55, "max_ms": 637.994, "p95_ms": 574.471}, "read_interval": {"avg_ms": 547.725, "count": 54, "max_ms": 646.755, "p95_ms": 583.279}, "triton_wait": {"avg_ms": 91.136, "count": 55, "max_ms": 128.926, "p95_ms": 111.782}, "visualize": {"avg_ms": 20.998, "count": 55, "max_ms": 26.742, "p95_ms": 22.802}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.603, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 22, "tracking_summary": 665}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5084.988, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 18, "h264_decode_error_count": 27, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:03:53+09:00", "last_error_summary": "[h264 @ 0x55b2cdf1fe00] error while decoding MB 12 43, bytestream -28", "last_exit_at": "2026-07-02T12:03:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 144.244, "last_started_at": "2026-07-02T12:01:30+09:00", "pid": 6433, "process": {"exists": false, "pid": 6433}, "restart_count": 23, "restart_count_per_hour": 16, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 24}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 605.282, "sample_count": 580, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 79887, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:03:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T12:03:54+09:00", "last_frame_read_age_sec": 0.559, "last_frame_read_at": "2026-07-02T12:03:53+09:00", "mjpeg": {"age_sec": 0.553, "exists": true, "mtime": "2026-07-02T12:03:53.876027+09:00", "mtime_epoch": 1782961433.876027, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206292}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:03:54+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1319.91, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 55, "pushed_frames": 55, "read_frames": 54, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.858, "pushed": 1.858, "read": 1.824}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 28.487, "count": 55, "max_ms": 42.971, "p95_ms": 32.449}, "gstreamer_push": {"avg_ms": 0.87, "count": 55, "max_ms": 0.966, "p95_ms": 0.93}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.371, "count": 55, "max_ms": 73.6, "p95_ms": 64.446}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 90.101, "count": 55, "max_ms": 128.914, "p95_ms": 110.215}, "yolo11_640_fire_11class": {"avg_ms": 46.53, "count": 55, "max_ms": 70.447, "p95_ms": 60.162}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.695, "count": 55, "max_ms": 125.139, "p95_ms": 106.305}}, "truncated_count": 0}, "processing_total": {"avg_ms": 538.217, "count": 55, "max_ms": 637.994, "p95_ms": 574.471}, "read_interval": {"avg_ms": 547.725, "count": 54, "max_ms": 646.755, "p95_ms": 583.279}, "triton_wait": {"avg_ms": 91.136, "count": 55, "max_ms": 128.926, "p95_ms": 111.782}, "visualize": {"avg_ms": 20.998, "count": 55, "max_ms": 26.742, "p95_ms": 22.802}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.606, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 22, "tracking_summary": 665}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5209.586, "limit_mb": 8796093022207.996}, "detail": {"pid": 6600, "restart_count": 24, "start_count": 25}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 18, "h264_decode_error_count": 27, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:03:53+09:00", "last_error_summary": "[h264 @ 0x55b2cdf1fe00] error while decoding MB 12 43, bytestream -28", "last_exit_at": "2026-07-02T12:03:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T12:03:57+09:00", "pid": 6600, "process": {"exists": true, "fd_count": 5, "hwm_mb": 149.848, "name": "ffmpeg", "pid": 6600, "rss_mb": 149.848, "state": "S (sleeping)", "threads": 106}, "restart_count": 24, "restart_count_per_hour": 17, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 25}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 606.038, "sample_count": 569, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 79887, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:03:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.97, "last_frame_pushed_at": "2026-07-02T12:03:54+09:00", "last_frame_read_age_sec": 5.524, "last_frame_read_at": "2026-07-02T12:03:53+09:00", "mjpeg": {"age_sec": 4.957, "exists": true, "mtime": "2026-07-02T12:03:54.436041+09:00", "mtime_epoch": 1782961434.436041, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206310}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:03:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1319.91, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 46, "pushed_frames": 46, "read_frames": 45, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.56, "pushed": 1.56, "read": 1.526}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 28.086, "count": 46, "max_ms": 32.449, "p95_ms": 30.8}, "gstreamer_push": {"avg_ms": 0.869, "count": 46, "max_ms": 0.966, "p95_ms": 0.93}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.057, "count": 46, "max_ms": 73.6, "p95_ms": 60.046}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 90.776, "count": 46, "max_ms": 128.914, "p95_ms": 107.701}, "yolo11_640_fire_11class": {"avg_ms": 46.091, "count": 46, "max_ms": 70.447, "p95_ms": 57.19}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 84.577, "count": 46, "max_ms": 124.995, "p95_ms": 105.272}}, "truncated_count": 0}, "processing_total": {"avg_ms": 535.254, "count": 46, "max_ms": 637.994, "p95_ms": 571.82}, "read_interval": {"avg_ms": 544.664, "count": 45, "max_ms": 646.755, "p95_ms": 582.375}, "triton_wait": {"avg_ms": 91.237, "count": 46, "max_ms": 128.926, "p95_ms": 107.712}, "visualize": {"avg_ms": 20.765, "count": 46, "max_ms": 22.802, "p95_ms": 22.492}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.49, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 22, "tracking_summary": 668}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5024.637, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[NULL @ 0x55de38f7c300] missing picture in access unit with size 39 [h264 @ 0x55de39ae9800] no frame! [vist#0:0/h264 @ 0x55de390e6680] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 19, "h264_decode_error_count": 28, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:06:14+09:00", "last_error_summary": "[NULL @ 0x55de38f7c300] missing picture in access unit with size 39 [h264 @ 0x55de39ae9800] no frame! [vist#0:0/h264 @ 0x55de390e6680] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T12:03:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 137.071, "last_started_at": "2026-07-02T12:03:57+09:00", "pid": 6600, "process": {"exists": false, "pid": 6600}, "restart_count": 24, "restart_count_per_hour": 17, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 25}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 605.282, "sample_count": 554, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 80165, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:06:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.056, "last_frame_pushed_at": "2026-07-02T12:06:14+09:00", "last_frame_read_age_sec": 0.048, "last_frame_read_at": "2026-07-02T12:06:14+09:00", "mjpeg": {"age_sec": 0.036, "exists": true, "mtime": "2026-07-02T12:06:14.427469+09:00", "mtime_epoch": 1782961574.427469, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 212192}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:06:14+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1319.91, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 61, "pushed_frames": 61, "read_frames": 61, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.054, "pushed": 2.054, "read": 2.054}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 27.672, "count": 60, "max_ms": 36.749, "p95_ms": 34.167}, "gstreamer_push": {"avg_ms": 0.867, "count": 61, "max_ms": 0.956, "p95_ms": 0.912}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.961, "count": 60, "max_ms": 68.602, "p95_ms": 61.545}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 83.652, "count": 60, "max_ms": 115.463, "p95_ms": 103.828}, "yolo11_640_fire_11class": {"avg_ms": 48.054, "count": 60, "max_ms": 70.431, "p95_ms": 62.75}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.421, "count": 60, "max_ms": 116.926, "p95_ms": 96.381}}, "truncated_count": 0}, "processing_total": {"avg_ms": 484.869, "count": 61, "max_ms": 529.037, "p95_ms": 518.167}, "read_interval": {"avg_ms": 494.076, "count": 61, "max_ms": 539.861, "p95_ms": 527.096}, "triton_wait": {"avg_ms": 84.805, "count": 60, "max_ms": 116.934, "p95_ms": 109.36}, "visualize": {"avg_ms": 19.87, "count": 60, "max_ms": 23.648, "p95_ms": 23.117}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.701, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 22, "tracking_summary": 668}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5023.461, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T12:06:14+09:00", "last_error_summary": "[NULL @ 0x55de38f7c300] missing picture in access unit with size 39 [h264 @ 0x55de39ae9800] no frame! [vist#0:0/h264 @ 0x55de390e6680] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 19, "h264_decode_error_count": 28, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:06:14+09:00", "last_error_summary": "[NULL @ 0x55de38f7c300] missing picture in access unit with size 39 [h264 @ 0x55de39ae9800] no frame! [vist#0:0/h264 @ 0x55de390e6680] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T12:06:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 137.495, "last_started_at": "2026-07-02T12:03:57+09:00", "pid": 6600, "process": {"exists": false, "pid": 6600}, "restart_count": 24, "restart_count_per_hour": 17, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 25}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 605.282, "sample_count": 554, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 80166, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:06:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T12:06:14+09:00", "last_frame_read_age_sec": 0.473, "last_frame_read_at": "2026-07-02T12:06:14+09:00", "mjpeg": {"age_sec": 0.46, "exists": true, "mtime": "2026-07-02T12:06:14.427469+09:00", "mtime_epoch": 1782961574.427469, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 212192}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:06:14+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1319.91, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 61, "pushed_frames": 61, "read_frames": 60, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.059, "pushed": 2.059, "read": 2.025}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 27.613, "count": 61, "max_ms": 36.749, "p95_ms": 34.167}, "gstreamer_push": {"avg_ms": 0.867, "count": 61, "max_ms": 0.956, "p95_ms": 0.912}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.983, "count": 61, "max_ms": 68.602, "p95_ms": 61.545}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 83.628, "count": 61, "max_ms": 115.463, "p95_ms": 103.828}, "yolo11_640_fire_11class": {"avg_ms": 47.965, "count": 61, "max_ms": 70.431, "p95_ms": 62.75}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.437, "count": 61, "max_ms": 116.926, "p95_ms": 96.381}}, "truncated_count": 0}, "processing_total": {"avg_ms": 484.555, "count": 61, "max_ms": 529.037, "p95_ms": 518.167}, "read_interval": {"avg_ms": 494.065, "count": 60, "max_ms": 539.861, "p95_ms": 527.096}, "triton_wait": {"avg_ms": 84.782, "count": 61, "max_ms": 116.934, "p95_ms": 109.36}, "visualize": {"avg_ms": 19.868, "count": 61, "max_ms": 23.648, "p95_ms": 23.117}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.631, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 22, "tracking_summary": 668}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5023.453, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 19, "h264_decode_error_count": 28, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:06:14+09:00", "last_error_summary": "[NULL @ 0x55de38f7c300] missing picture in access unit with size 39 [h264 @ 0x55de39ae9800] no frame! [vist#0:0/h264 @ 0x55de390e6680] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T12:06:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 137.499, "last_started_at": "2026-07-02T12:03:57+09:00", "pid": 6600, "process": {"exists": false, "pid": 6600}, "restart_count": 24, "restart_count_per_hour": 17, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 25}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 605.282, "sample_count": 554, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 80166, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:06:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-02T12:06:14+09:00", "last_frame_read_age_sec": 0.477, "last_frame_read_at": "2026-07-02T12:06:14+09:00", "mjpeg": {"age_sec": 0.464, "exists": true, "mtime": "2026-07-02T12:06:14.427469+09:00", "mtime_epoch": 1782961574.427469, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 212192}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:06:14+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1319.91, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 61, "pushed_frames": 61, "read_frames": 60, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.058, "pushed": 2.058, "read": 2.025}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 27.613, "count": 61, "max_ms": 36.749, "p95_ms": 34.167}, "gstreamer_push": {"avg_ms": 0.867, "count": 61, "max_ms": 0.956, "p95_ms": 0.912}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.983, "count": 61, "max_ms": 68.602, "p95_ms": 61.545}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 83.628, "count": 61, "max_ms": 115.463, "p95_ms": 103.828}, "yolo11_640_fire_11class": {"avg_ms": 47.965, "count": 61, "max_ms": 70.431, "p95_ms": 62.75}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.437, "count": 61, "max_ms": 116.926, "p95_ms": 96.381}}, "truncated_count": 0}, "processing_total": {"avg_ms": 484.555, "count": 61, "max_ms": 529.037, "p95_ms": 518.167}, "read_interval": {"avg_ms": 494.065, "count": 60, "max_ms": 539.861, "p95_ms": 527.096}, "triton_wait": {"avg_ms": 84.782, "count": 61, "max_ms": 116.934, "p95_ms": 109.36}, "visualize": {"avg_ms": 19.868, "count": 61, "max_ms": 23.648, "p95_ms": 23.117}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.635, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 22, "tracking_summary": 668}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5134.445, "limit_mb": 8796093022207.996}, "detail": {"pid": 6707, "restart_count": 25, "start_count": 26}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 19, "h264_decode_error_count": 28, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:06:14+09:00", "last_error_summary": "[NULL @ 0x55de38f7c300] missing picture in access unit with size 39 [h264 @ 0x55de39ae9800] no frame! [vist#0:0/h264 @ 0x55de390e6680] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T12:06:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T12:06:16+09:00", "pid": 6707, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.961, "name": "ffmpeg", "pid": 6707, "rss_mb": 138.961, "state": "S (sleeping)", "threads": 106}, "restart_count": 25, "restart_count_per_hour": 18, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 26}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 606.038, "sample_count": 545, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 80166, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:06:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.114, "last_frame_pushed_at": "2026-07-02T12:06:14+09:00", "last_frame_read_age_sec": 4.583, "last_frame_read_at": "2026-07-02T12:06:14+09:00", "mjpeg": {"age_sec": 4.095, "exists": true, "mtime": "2026-07-02T12:06:14.903481+09:00", "mtime_epoch": 1782961574.9034805, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 212126}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:06:19+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1319.91, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 53, "pushed_frames": 53, "read_frames": 52, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.778, "pushed": 1.778, "read": 1.745}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 27.867, "count": 52, "max_ms": 36.749, "p95_ms": 34.225}, "gstreamer_push": {"avg_ms": 0.866, "count": 53, "max_ms": 0.956, "p95_ms": 0.912}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.431, "count": 52, "max_ms": 68.602, "p95_ms": 63.256}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 84.337, "count": 52, "max_ms": 115.463, "p95_ms": 109.346}, "yolo11_640_fire_11class": {"avg_ms": 48.267, "count": 52, "max_ms": 70.431, "p95_ms": 63.776}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.096, "count": 52, "max_ms": 116.926, "p95_ms": 105.143}}, "truncated_count": 0}, "processing_total": {"avg_ms": 484.877, "count": 53, "max_ms": 529.037, "p95_ms": 521.654}, "read_interval": {"avg_ms": 494.46, "count": 52, "max_ms": 539.861, "p95_ms": 532.127}, "triton_wait": {"avg_ms": 85.646, "count": 52, "max_ms": 116.934, "p95_ms": 109.398}, "visualize": {"avg_ms": 19.924, "count": 52, "max_ms": 23.648, "p95_ms": 23.424}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.807, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 24, "tracking_summary": 670}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5218.547, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55eb3fe41b40] error while decoding MB 27 37, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 20, "h264_decode_error_count": 29, "h264_decode_error_count_per_hour": 16, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:08:45+09:00", "last_error_summary": "[h264 @ 0x55eb3fe41b40] error while decoding MB 27 37, bytestream -7", "last_exit_at": "2026-07-02T12:06:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 148.334, "last_started_at": "2026-07-02T12:06:16+09:00", "pid": 6707, "process": {"exists": false, "pid": 6707}, "restart_count": 25, "restart_count_per_hour": 18, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 26}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 665.328, "sample_count": 550, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 80420, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:08:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-02T12:08:45+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-02T12:08:45+09:00", "mjpeg": {"age_sec": 0.037, "exists": true, "mtime": "2026-07-02T12:08:45.295016+09:00", "mtime_epoch": 1782961725.295016, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204898}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:08:45+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1347.293, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": 31.759, "recorders": [{"age_sec": 31.759, "frame_count": 94, "is_recording": true, "is_stopping": false, "pid": 6814, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.82, "name": "ffmpeg", "pid": 6814, "rss_mb": 186.82, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 6, "returncode": null, "running": true, "started_at": "2026-07-02T12:08:13+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_120813_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [6814], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 43, "pushed_frames": 43, "read_frames": 43, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.446, "pushed": 1.446, "read": 1.446}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 40.064, "count": 42, "max_ms": 46.754, "p95_ms": 44.901}, "gstreamer_push": {"avg_ms": 0.886, "count": 43, "max_ms": 0.998, "p95_ms": 0.954}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.052, "count": 42, "max_ms": 73.666, "p95_ms": 70.081}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 86.723, "count": 42, "max_ms": 118.434, "p95_ms": 114.602}, "yolo11_640_fire_11class": {"avg_ms": 48.286, "count": 42, "max_ms": 67.381, "p95_ms": 62.537}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.287, "count": 42, "max_ms": 106.703, "p95_ms": 100.353}}, "truncated_count": 0}, "processing_total": {"avg_ms": 700.297, "count": 43, "max_ms": 975.356, "p95_ms": 957.682}, "read_interval": {"avg_ms": 709.683, "count": 43, "max_ms": 985.255, "p95_ms": 967.707}, "triton_wait": {"avg_ms": 87.888, "count": 42, "max_ms": 118.445, "p95_ms": 114.614}, "visualize": {"avg_ms": 27.453, "count": 42, "max_ms": 34.522, "p95_ms": 31.938}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.732, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 25, "tracking_summary": 670}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5218.355, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T12:08:45+09:00", "last_error_summary": "[h264 @ 0x55eb3fe41b40] error while decoding MB 27 37, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 20, "h264_decode_error_count": 29, "h264_decode_error_count_per_hour": 16, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:08:45+09:00", "last_error_summary": "[h264 @ 0x55eb3fe41b40] error while decoding MB 27 37, bytestream -7", "last_exit_at": "2026-07-02T12:08:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 149.011, "last_started_at": "2026-07-02T12:06:16+09:00", "pid": 6707, "process": {"exists": false, "pid": 6707}, "restart_count": 25, "restart_count_per_hour": 18, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 26}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 666.862, "sample_count": 549, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 80421, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:08:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T12:08:46+09:00", "last_frame_read_age_sec": 0.728, "last_frame_read_at": "2026-07-02T12:08:45+09:00", "mjpeg": {"age_sec": 0.713, "exists": true, "mtime": "2026-07-02T12:08:45.295016+09:00", "mtime_epoch": 1782961725.295016, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204898}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:08:46+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1347.293, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": 32.434, "recorders": [{"age_sec": 32.434, "frame_count": 95, "is_recording": true, "is_stopping": false, "pid": 6814, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.82, "name": "ffmpeg", "pid": 6814, "rss_mb": 186.82, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 5, "returncode": null, "running": true, "started_at": "2026-07-02T12:08:13+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_120813_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [6814], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 43, "pushed_frames": 43, "read_frames": 42, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.446, "pushed": 1.446, "read": 1.413}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 39.896, "count": 42, "max_ms": 46.754, "p95_ms": 44.901}, "gstreamer_push": {"avg_ms": 0.885, "count": 43, "max_ms": 0.998, "p95_ms": 0.954}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.225, "count": 42, "max_ms": 73.666, "p95_ms": 70.081}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 86.646, "count": 42, "max_ms": 118.434, "p95_ms": 114.602}, "yolo11_640_fire_11class": {"avg_ms": 48.308, "count": 42, "max_ms": 67.381, "p95_ms": 62.537}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.457, "count": 42, "max_ms": 106.703, "p95_ms": 100.353}}, "truncated_count": 0}, "processing_total": {"avg_ms": 697.65, "count": 43, "max_ms": 975.356, "p95_ms": 957.682}, "read_interval": {"avg_ms": 706.488, "count": 42, "max_ms": 985.255, "p95_ms": 967.707}, "triton_wait": {"avg_ms": 87.94, "count": 42, "max_ms": 118.445, "p95_ms": 114.614}, "visualize": {"avg_ms": 27.465, "count": 42, "max_ms": 34.522, "p95_ms": 31.938}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.731, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 25, "tracking_summary": 670}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5218.348, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 20, "h264_decode_error_count": 29, "h264_decode_error_count_per_hour": 16, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:08:45+09:00", "last_error_summary": "[h264 @ 0x55eb3fe41b40] error while decoding MB 27 37, bytestream -7", "last_exit_at": "2026-07-02T12:08:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 149.013, "last_started_at": "2026-07-02T12:06:16+09:00", "pid": 6707, "process": {"exists": false, "pid": 6707}, "restart_count": 25, "restart_count_per_hour": 18, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 26}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 666.862, "sample_count": 549, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 80421, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:08:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T12:08:46+09:00", "last_frame_read_age_sec": 0.731, "last_frame_read_at": "2026-07-02T12:08:45+09:00", "mjpeg": {"age_sec": 0.716, "exists": true, "mtime": "2026-07-02T12:08:45.295016+09:00", "mtime_epoch": 1782961725.295016, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204898}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:08:46+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1347.293, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": 32.437, "recorders": [{"age_sec": 32.437, "frame_count": 95, "is_recording": true, "is_stopping": false, "pid": 6814, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.82, "name": "ffmpeg", "pid": 6814, "rss_mb": 186.82, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 5, "returncode": null, "running": true, "started_at": "2026-07-02T12:08:13+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_120813_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [6814], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 43, "pushed_frames": 43, "read_frames": 42, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.446, "pushed": 1.446, "read": 1.413}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 39.896, "count": 42, "max_ms": 46.754, "p95_ms": 44.901}, "gstreamer_push": {"avg_ms": 0.885, "count": 43, "max_ms": 0.998, "p95_ms": 0.954}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.225, "count": 42, "max_ms": 73.666, "p95_ms": 70.081}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 86.646, "count": 42, "max_ms": 118.434, "p95_ms": 114.602}, "yolo11_640_fire_11class": {"avg_ms": 48.308, "count": 42, "max_ms": 67.381, "p95_ms": 62.537}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 82.457, "count": 42, "max_ms": 106.703, "p95_ms": 100.353}}, "truncated_count": 0}, "processing_total": {"avg_ms": 697.65, "count": 43, "max_ms": 975.356, "p95_ms": 957.682}, "read_interval": {"avg_ms": 706.488, "count": 42, "max_ms": 985.255, "p95_ms": 967.707}, "triton_wait": {"avg_ms": 87.94, "count": 42, "max_ms": 118.445, "p95_ms": 114.614}, "visualize": {"avg_ms": 27.465, "count": 42, "max_ms": 34.522, "p95_ms": 31.938}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.734, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 25, "tracking_summary": 670}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5331.859, "limit_mb": 8796093022207.996}, "detail": {"pid": 6874, "restart_count": 26, "start_count": 27}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 20, "h264_decode_error_count": 29, "h264_decode_error_count_per_hour": 16, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:08:45+09:00", "last_error_summary": "[h264 @ 0x55eb3fe41b40] error while decoding MB 27 37, bytestream -7", "last_exit_at": "2026-07-02T12:08:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T12:08:48+09:00", "pid": 6874, "process": {"exists": true, "fd_count": 5, "hwm_mb": 149.66, "name": "ffmpeg", "pid": 6874, "rss_mb": 149.66, "state": "S (sleeping)", "threads": 106}, "restart_count": 26, "restart_count_per_hour": 19, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 27}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 666.862, "sample_count": 542, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 80421, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:08:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.407, "last_frame_pushed_at": "2026-07-02T12:08:46+09:00", "last_frame_read_age_sec": 5.131, "last_frame_read_at": "2026-07-02T12:08:45+09:00", "mjpeg": {"age_sec": 4.384, "exists": true, "mtime": "2026-07-02T12:08:46.027033+09:00", "mtime_epoch": 1782961726.0270329, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204975}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:08:50+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1347.293, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": 36.838, "recorders": [{"age_sec": 36.838, "frame_count": 95, "is_recording": true, "is_stopping": false, "pid": 6814, "process": {"exists": true, "fd_count": 5, "hwm_mb": 186.82, "name": "ffmpeg", "pid": 6814, "rss_mb": 186.82, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 5, "returncode": null, "running": true, "started_at": "2026-07-02T12:08:13+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_120813_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [6814], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 36, "pushed_frames": 36, "read_frames": 35, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.226, "pushed": 1.226, "read": 1.192}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 39.925, "count": 36, "max_ms": 46.754, "p95_ms": 45.463}, "gstreamer_push": {"avg_ms": 0.879, "count": 36, "max_ms": 0.998, "p95_ms": 0.925}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.586, "count": 36, "max_ms": 73.666, "p95_ms": 70.625}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 87.238, "count": 36, "max_ms": 118.434, "p95_ms": 114.803}, "yolo11_640_fire_11class": {"avg_ms": 48.935, "count": 36, "max_ms": 67.381, "p95_ms": 63.821}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 83.192, "count": 36, "max_ms": 106.703, "p95_ms": 101.533}}, "truncated_count": 0}, "processing_total": {"avg_ms": 702.218, "count": 36, "max_ms": 975.356, "p95_ms": 960.018}, "read_interval": {"avg_ms": 711.114, "count": 35, "max_ms": 985.255, "p95_ms": 969.087}, "triton_wait": {"avg_ms": 88.271, "count": 36, "max_ms": 118.445, "p95_ms": 114.816}, "visualize": {"avg_ms": 27.404, "count": 36, "max_ms": 34.522, "p95_ms": 32.098}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.356, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 25, "tracking_summary": 672}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5033.648, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5579c5e23600] error while decoding MB 7 41, bytestream -15", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 21, "h264_decode_error_count": 30, "h264_decode_error_count_per_hour": 17, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:11:14+09:00", "last_error_summary": "[h264 @ 0x5579c5e23600] error while decoding MB 7 41, bytestream -15", "last_exit_at": "2026-07-02T12:08:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 145.978, "last_started_at": "2026-07-02T12:08:48+09:00", "pid": 6874, "process": {"exists": false, "pid": 6874}, "restart_count": 26, "restart_count_per_hour": 19, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 27}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 680.111, "sample_count": 495, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 80659, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:11:13+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-02T12:11:14+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-02T12:11:14+09:00", "mjpeg": {"age_sec": 0.038, "exists": true, "mtime": "2026-07-02T12:11:14.350408+09:00", "mtime_epoch": 1782961874.3504078, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 215932}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:11:14+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1321.059, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 51, "pushed_frames": 51, "read_frames": 51, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.712, "pushed": 1.712, "read": 1.712}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 36.507, "count": 50, "max_ms": 56.469, "p95_ms": 48.573}, "gstreamer_push": {"avg_ms": 0.873, "count": 51, "max_ms": 0.995, "p95_ms": 0.965}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.036, "count": 50, "max_ms": 73.978, "p95_ms": 66.554}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 85.671, "count": 50, "max_ms": 113.999, "p95_ms": 103.39}, "yolo11_640_fire_11class": {"avg_ms": 50.864, "count": 50, "max_ms": 69.695, "p95_ms": 68.237}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 79.351, "count": 50, "max_ms": 106.944, "p95_ms": 101.837}}, "truncated_count": 0}, "processing_total": {"avg_ms": 586.013, "count": 51, "max_ms": 692.608, "p95_ms": 636.221}, "read_interval": {"avg_ms": 595.57, "count": 51, "max_ms": 702.318, "p95_ms": 646.718}, "triton_wait": {"avg_ms": 86.511, "count": 50, "max_ms": 114.011, "p95_ms": 106.78}, "visualize": {"avg_ms": 18.801, "count": 50, "max_ms": 26.053, "p95_ms": 21.751}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.793, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 26, "tracking_summary": 672}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5189.082, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T12:11:14+09:00", "last_error_summary": "[h264 @ 0x5579c5e23600] error while decoding MB 7 41, bytestream -15", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 21, "h264_decode_error_count": 30, "h264_decode_error_count_per_hour": 17, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:11:14+09:00", "last_error_summary": "[h264 @ 0x5579c5e23600] error while decoding MB 7 41, bytestream -15", "last_exit_at": "2026-07-02T12:11:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 147.271, "last_started_at": "2026-07-02T12:08:48+09:00", "pid": 6874, "process": {"exists": false, "pid": 6874}, "restart_count": 26, "restart_count_per_hour": 19, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 27}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 680.205, "sample_count": 494, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 80660, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:11:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T12:11:15+09:00", "last_frame_read_age_sec": 1.343, "last_frame_read_at": "2026-07-02T12:11:14+09:00", "mjpeg": {"age_sec": 1.331, "exists": true, "mtime": "2026-07-02T12:11:14.350408+09:00", "mtime_epoch": 1782961874.3504078, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 215932}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:11:15+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1321.059, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": 1.159, "recorders": [{"age_sec": 1.159, "frame_count": 51, "is_recording": true, "is_stopping": false, "pid": 6981, "process": {"exists": true, "fd_count": 5, "hwm_mb": 187.164, "name": "ffmpeg", "pid": 6981, "rss_mb": 187.164, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 49, "returncode": null, "running": true, "started_at": "2026-07-02T12:11:14+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_121114_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [6981], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 50, "pushed_frames": 50, "read_frames": 49, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.675, "pushed": 1.675, "read": 1.642}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 35.778, "count": 49, "max_ms": 50.297, "p95_ms": 45.218}, "gstreamer_push": {"avg_ms": 0.873, "count": 50, "max_ms": 0.995, "p95_ms": 0.965}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.183, "count": 49, "max_ms": 73.978, "p95_ms": 66.554}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 86.815, "count": 49, "max_ms": 113.999, "p95_ms": 106.767}, "yolo11_640_fire_11class": {"avg_ms": 51.376, "count": 49, "max_ms": 69.695, "p95_ms": 68.237}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.448, "count": 49, "max_ms": 108.546, "p95_ms": 104.628}}, "truncated_count": 0}, "processing_total": {"avg_ms": 599.585, "count": 50, "max_ms": 1338.129, "p95_ms": 684.725}, "read_interval": {"avg_ms": 594.084, "count": 49, "max_ms": 702.318, "p95_ms": 646.718}, "triton_wait": {"avg_ms": 87.671, "count": 49, "max_ms": 114.011, "p95_ms": 106.956}, "visualize": {"avg_ms": 18.868, "count": 49, "max_ms": 26.053, "p95_ms": 21.751}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.849, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 26, "tracking_summary": 672}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5189.09, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 21, "h264_decode_error_count": 30, "h264_decode_error_count_per_hour": 17, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:11:14+09:00", "last_error_summary": "[h264 @ 0x5579c5e23600] error while decoding MB 7 41, bytestream -15", "last_exit_at": "2026-07-02T12:11:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 147.274, "last_started_at": "2026-07-02T12:08:48+09:00", "pid": 6874, "process": {"exists": false, "pid": 6874}, "restart_count": 26, "restart_count_per_hour": 19, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 27}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 680.205, "sample_count": 494, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 80660, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:11:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T12:11:15+09:00", "last_frame_read_age_sec": 1.346, "last_frame_read_at": "2026-07-02T12:11:14+09:00", "mjpeg": {"age_sec": 1.334, "exists": true, "mtime": "2026-07-02T12:11:14.350408+09:00", "mtime_epoch": 1782961874.3504078, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 215932}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:11:15+09:00", "process": {"exists": true, "fd_count": 121, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1321.059, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": 1.162, "recorders": [{"age_sec": 1.162, "frame_count": 51, "is_recording": true, "is_stopping": false, "pid": 6981, "process": {"exists": true, "fd_count": 5, "hwm_mb": 187.164, "name": "ffmpeg", "pid": 6981, "rss_mb": 187.164, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 49, "returncode": null, "running": true, "started_at": "2026-07-02T12:11:14+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_121114_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [6981], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 50, "pushed_frames": 50, "read_frames": 49, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.675, "pushed": 1.675, "read": 1.641}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 35.778, "count": 49, "max_ms": 50.297, "p95_ms": 45.218}, "gstreamer_push": {"avg_ms": 0.873, "count": 50, "max_ms": 0.995, "p95_ms": 0.965}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.183, "count": 49, "max_ms": 73.978, "p95_ms": 66.554}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 86.815, "count": 49, "max_ms": 113.999, "p95_ms": 106.767}, "yolo11_640_fire_11class": {"avg_ms": 51.376, "count": 49, "max_ms": 69.695, "p95_ms": 68.237}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.448, "count": 49, "max_ms": 108.546, "p95_ms": 104.628}}, "truncated_count": 0}, "processing_total": {"avg_ms": 599.585, "count": 50, "max_ms": 1338.129, "p95_ms": 684.725}, "read_interval": {"avg_ms": 594.084, "count": 49, "max_ms": 702.318, "p95_ms": 646.718}, "triton_wait": {"avg_ms": 87.671, "count": 49, "max_ms": 114.011, "p95_ms": 106.956}, "visualize": {"avg_ms": 18.868, "count": 49, "max_ms": 26.053, "p95_ms": 21.751}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.852, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 26, "tracking_summary": 672}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5294.5, "limit_mb": 8796093022207.996}, "detail": {"pid": 7041, "restart_count": 27, "start_count": 28}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 21, "h264_decode_error_count": 30, "h264_decode_error_count_per_hour": 17, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:11:14+09:00", "last_error_summary": "[h264 @ 0x5579c5e23600] error while decoding MB 7 41, bytestream -15", "last_exit_at": "2026-07-02T12:11:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T12:11:18+09:00", "pid": 7041, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.504, "name": "ffmpeg", "pid": 7041, "rss_mb": 136.504, "state": "S (sleeping)", "threads": 106}, "restart_count": 27, "restart_count_per_hour": 20, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 28}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 680.205, "sample_count": 492, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 80660, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:11:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.457, "last_frame_pushed_at": "2026-07-02T12:11:15+09:00", "last_frame_read_age_sec": 5.796, "last_frame_read_at": "2026-07-02T12:11:14+09:00", "mjpeg": {"age_sec": 4.44, "exists": true, "mtime": "2026-07-02T12:11:15.694438+09:00", "mtime_epoch": 1782961875.694438, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 216175}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:11:20+09:00", "process": {"exists": true, "fd_count": 122, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1321.059, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 1, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": 5.613, "recorders": [{"age_sec": 5.613, "frame_count": 51, "is_recording": true, "is_stopping": false, "pid": 6981, "process": {"exists": true, "fd_count": 5, "hwm_mb": 187.164, "name": "ffmpeg", "pid": 6981, "rss_mb": 187.164, "state": "S (sleeping)", "threads": 60}, "remaining_frames": 49, "returncode": null, "running": true, "started_at": "2026-07-02T12:11:14+09:00", "total_frames_to_record": 100, "videoname": "/data/playlist/accident/02/20260702_121114_no_hardhat_persistent"}], "running_ffmpeg_count": 1, "running_pids": [6981], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 42, "pushed_frames": 42, "read_frames": 41, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.423, "pushed": 1.423, "read": 1.389}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 35.153, "count": 42, "max_ms": 44.953, "p95_ms": 41.012}, "gstreamer_push": {"avg_ms": 0.876, "count": 42, "max_ms": 0.995, "p95_ms": 0.965}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.699, "count": 42, "max_ms": 73.978, "p95_ms": 65.307}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 87.106, "count": 42, "max_ms": 113.999, "p95_ms": 106.767}, "yolo11_640_fire_11class": {"avg_ms": 51.81, "count": 42, "max_ms": 69.695, "p95_ms": 68.237}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 80.343, "count": 42, "max_ms": 108.546, "p95_ms": 101.837}}, "truncated_count": 0}, "processing_total": {"avg_ms": 601.998, "count": 42, "max_ms": 1338.129, "p95_ms": 684.725}, "read_interval": {"avg_ms": 593.597, "count": 41, "max_ms": 702.318, "p95_ms": 646.718}, "triton_wait": {"avg_ms": 87.995, "count": 42, "max_ms": 114.011, "p95_ms": 106.78}, "visualize": {"avg_ms": 18.989, "count": 42, "max_ms": 26.053, "p95_ms": 21.414}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.521, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 30, "tracking_summary": 673}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5051.477, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T12:11:14+09:00", "last_error_summary": "[h264 @ 0x5579c5e23600] error while decoding MB 7 41, bytestream -15", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 21, "h264_decode_error_count": 30, "h264_decode_error_count_per_hour": 17, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:11:14+09:00", "last_error_summary": "[h264 @ 0x5579c5e23600] error while decoding MB 7 41, bytestream -15", "last_exit_at": "2026-07-02T12:13:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 133.156, "last_started_at": "2026-07-02T12:11:18+09:00", "pid": 7041, "process": {"exists": false, "pid": 7041}, "restart_count": 27, "restart_count_per_hour": 20, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 28}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1037.646, "sample_count": 407, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 80805, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:13:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.002, "last_frame_pushed_at": "2026-07-02T12:13:31+09:00", "last_frame_read_age_sec": 0.799, "last_frame_read_at": "2026-07-02T12:13:30+09:00", "mjpeg": {"age_sec": 0.792, "exists": true, "mtime": "2026-07-02T12:13:30.497432+09:00", "mtime_epoch": 1782962010.4974325, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 208287}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:13:31+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1325.492, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 38, "pushed_frames": 38, "read_frames": 37, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.296, "pushed": 1.296, "read": 1.262}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 46.135, "count": 38, "max_ms": 61.696, "p95_ms": 55.089}, "gstreamer_push": {"avg_ms": 0.888, "count": 38, "max_ms": 1.299, "p95_ms": 1.121}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.77, "count": 38, "max_ms": 71.003, "p95_ms": 68.457}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 90.51, "count": 38, "max_ms": 109.169, "p95_ms": 106.499}, "yolo11_640_fire_11class": {"avg_ms": 52.085, "count": 38, "max_ms": 74.045, "p95_ms": 68.445}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 93.186, "count": 38, "max_ms": 121.184, "p95_ms": 116.318}}, "truncated_count": 0}, "processing_total": {"avg_ms": 784.223, "count": 38, "max_ms": 862.408, "p95_ms": 830.966}, "read_interval": {"avg_ms": 793.193, "count": 37, "max_ms": 871.502, "p95_ms": 840.436}, "triton_wait": {"avg_ms": 95.515, "count": 38, "max_ms": 121.197, "p95_ms": 116.331}, "visualize": {"avg_ms": 23.236, "count": 38, "max_ms": 27.173, "p95_ms": 25.771}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.325, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 30, "tracking_summary": 673}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5051.562, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 21, "h264_decode_error_count": 30, "h264_decode_error_count_per_hour": 17, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:11:14+09:00", "last_error_summary": "[h264 @ 0x5579c5e23600] error while decoding MB 7 41, bytestream -15", "last_exit_at": "2026-07-02T12:13:31+09:00", "last_returncode": 0, "last_stable_duration_sec": 133.158, "last_started_at": "2026-07-02T12:11:18+09:00", "pid": 7041, "process": {"exists": false, "pid": 7041}, "restart_count": 27, "restart_count_per_hour": 20, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 28}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1037.646, "sample_count": 407, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 80805, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:13:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T12:13:31+09:00", "last_frame_read_age_sec": 0.801, "last_frame_read_at": "2026-07-02T12:13:30+09:00", "mjpeg": {"age_sec": 0.794, "exists": true, "mtime": "2026-07-02T12:13:30.497432+09:00", "mtime_epoch": 1782962010.4974325, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 208287}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:13:31+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1325.492, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 38, "pushed_frames": 38, "read_frames": 37, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.296, "pushed": 1.296, "read": 1.262}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 46.135, "count": 38, "max_ms": 61.696, "p95_ms": 55.089}, "gstreamer_push": {"avg_ms": 0.888, "count": 38, "max_ms": 1.299, "p95_ms": 1.121}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.77, "count": 38, "max_ms": 71.003, "p95_ms": 68.457}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 90.51, "count": 38, "max_ms": 109.169, "p95_ms": 106.499}, "yolo11_640_fire_11class": {"avg_ms": 52.085, "count": 38, "max_ms": 74.045, "p95_ms": 68.445}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 93.186, "count": 38, "max_ms": 121.184, "p95_ms": 116.318}}, "truncated_count": 0}, "processing_total": {"avg_ms": 784.223, "count": 38, "max_ms": 862.408, "p95_ms": 830.966}, "read_interval": {"avg_ms": 793.193, "count": 37, "max_ms": 871.502, "p95_ms": 840.436}, "triton_wait": {"avg_ms": 95.515, "count": 38, "max_ms": 121.197, "p95_ms": 116.331}, "visualize": {"avg_ms": 23.236, "count": 38, "max_ms": 27.173, "p95_ms": 25.771}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.327, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 30, "tracking_summary": 673}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5165.652, "limit_mb": 8796093022207.996}, "detail": {"pid": 7148, "restart_count": 28, "start_count": 29}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 21, "h264_decode_error_count": 30, "h264_decode_error_count_per_hour": 17, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:11:14+09:00", "last_error_summary": "[h264 @ 0x5579c5e23600] error while decoding MB 7 41, bytestream -15", "last_exit_at": "2026-07-02T12:13:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T12:13:33+09:00", "pid": 7148, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.574, "name": "ffmpeg", "pid": 7148, "rss_mb": 136.574, "state": "S (sleeping)", "threads": 106}, "restart_count": 28, "restart_count_per_hour": 21, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 29}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 1037.646, "sample_count": 401, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 80805, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:13:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.119, "last_frame_pushed_at": "2026-07-02T12:13:31+09:00", "last_frame_read_age_sec": 4.916, "last_frame_read_at": "2026-07-02T12:13:30+09:00", "mjpeg": {"age_sec": 4.101, "exists": true, "mtime": "2026-07-02T12:13:31.305450+09:00", "mtime_epoch": 1782962011.3054502, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 207906}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:13:35+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1325.492, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 33, "pushed_frames": 33, "read_frames": 32, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.123, "pushed": 1.123, "read": 1.089}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 45.981, "count": 32, "max_ms": 55.089, "p95_ms": 53.032}, "gstreamer_push": {"avg_ms": 0.882, "count": 33, "max_ms": 1.299, "p95_ms": 0.987}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.057, "count": 32, "max_ms": 71.003, "p95_ms": 68.457}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 90.256, "count": 32, "max_ms": 106.499, "p95_ms": 101.084}, "yolo11_640_fire_11class": {"avg_ms": 51.606, "count": 32, "max_ms": 74.045, "p95_ms": 68.445}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 91.809, "count": 32, "max_ms": 121.184, "p95_ms": 116.318}}, "truncated_count": 0}, "processing_total": {"avg_ms": 780.402, "count": 33, "max_ms": 830.966, "p95_ms": 830.721}, "read_interval": {"avg_ms": 789.127, "count": 32, "max_ms": 840.436, "p95_ms": 839.322}, "triton_wait": {"avg_ms": 94.417, "count": 32, "max_ms": 121.197, "p95_ms": 116.331}, "visualize": {"avg_ms": 23.121, "count": 32, "max_ms": 27.173, "p95_ms": 25.427}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.375, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 30, "tracking_summary": 676}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5073.715, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55a9b9ca5b80] error while decoding MB 7 41, bytestream -17", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 22, "h264_decode_error_count": 31, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:15:44+09:00", "last_error_summary": "[h264 @ 0x55a9b9ca5b80] error while decoding MB 7 41, bytestream -17", "last_exit_at": "2026-07-02T12:13:33+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.001, "last_started_at": "2026-07-02T12:13:33+09:00", "pid": 7148, "process": {"exists": false, "pid": 7148}, "restart_count": 28, "restart_count_per_hour": 21, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 29}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 996.036, "sample_count": 519, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 81127, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:15:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-02T12:15:44+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-02T12:15:44+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-02T12:15:44.364354+09:00", "mtime_epoch": 1782962144.3643541, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199963}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:15:44+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1322.727, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 76, "pushed_frames": 76, "read_frames": 76, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.534, "pushed": 2.534, "read": 2.534}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 21.318, "count": 75, "max_ms": 27.414, "p95_ms": 24.2}, "gstreamer_push": {"avg_ms": 0.872, "count": 76, "max_ms": 1.048, "p95_ms": 0.924}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.737, "count": 75, "max_ms": 70.644, "p95_ms": 67.63}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.5, "count": 75, "max_ms": 106.569, "p95_ms": 98.983}, "yolo11_640_fire_11class": {"avg_ms": 49.336, "count": 75, "max_ms": 69.433, "p95_ms": 64.214}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 76.943, "count": 75, "max_ms": 109.988, "p95_ms": 102.869}}, "truncated_count": 0}, "processing_total": {"avg_ms": 390.455, "count": 76, "max_ms": 453.568, "p95_ms": 436.519}, "read_interval": {"avg_ms": 399.596, "count": 76, "max_ms": 462.616, "p95_ms": 444.497}, "triton_wait": {"avg_ms": 78.518, "count": 75, "max_ms": 110.001, "p95_ms": 106.582}, "visualize": {"avg_ms": 16.814, "count": 75, "max_ms": 21.385, "p95_ms": 18.73}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 30, "tracking_summary": 676}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5072.82, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T12:15:44+09:00", "last_error_summary": "[h264 @ 0x55a9b9ca5b80] error while decoding MB 7 41, bytestream -17", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 22, "h264_decode_error_count": 31, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:15:44+09:00", "last_error_summary": "[h264 @ 0x55a9b9ca5b80] error while decoding MB 7 41, bytestream -17", "last_exit_at": "2026-07-02T12:15:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.337, "last_started_at": "2026-07-02T12:13:33+09:00", "pid": 7148, "process": {"exists": false, "pid": 7148}, "restart_count": 28, "restart_count_per_hour": 21, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 29}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 996.036, "sample_count": 519, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 81128, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:15:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T12:15:44+09:00", "last_frame_read_age_sec": 0.387, "last_frame_read_at": "2026-07-02T12:15:44+09:00", "mjpeg": {"age_sec": 0.378, "exists": true, "mtime": "2026-07-02T12:15:44.364354+09:00", "mtime_epoch": 1782962144.3643541, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199963}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:15:44+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1322.727, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 76, "pushed_frames": 76, "read_frames": 75, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.541, "pushed": 2.541, "read": 2.508}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 21.249, "count": 75, "max_ms": 27.414, "p95_ms": 24.2}, "gstreamer_push": {"avg_ms": 0.873, "count": 76, "max_ms": 1.048, "p95_ms": 0.924}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.858, "count": 75, "max_ms": 70.644, "p95_ms": 67.63}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.833, "count": 75, "max_ms": 106.569, "p95_ms": 98.983}, "yolo11_640_fire_11class": {"avg_ms": 49.485, "count": 75, "max_ms": 69.433, "p95_ms": 64.214}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.161, "count": 75, "max_ms": 109.988, "p95_ms": 102.869}}, "truncated_count": 0}, "processing_total": {"avg_ms": 389.925, "count": 76, "max_ms": 453.568, "p95_ms": 436.519}, "read_interval": {"avg_ms": 399.146, "count": 75, "max_ms": 462.616, "p95_ms": 444.497}, "triton_wait": {"avg_ms": 78.851, "count": 75, "max_ms": 110.001, "p95_ms": 106.582}, "visualize": {"avg_ms": 16.783, "count": 75, "max_ms": 21.385, "p95_ms": 18.73}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.904, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 30, "tracking_summary": 676}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5072.812, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 22, "h264_decode_error_count": 31, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:15:44+09:00", "last_error_summary": "[h264 @ 0x55a9b9ca5b80] error while decoding MB 7 41, bytestream -17", "last_exit_at": "2026-07-02T12:15:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.34, "last_started_at": "2026-07-02T12:13:33+09:00", "pid": 7148, "process": {"exists": false, "pid": 7148}, "restart_count": 28, "restart_count_per_hour": 21, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 29}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 996.036, "sample_count": 519, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 81128, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:15:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T12:15:44+09:00", "last_frame_read_age_sec": 0.39, "last_frame_read_at": "2026-07-02T12:15:44+09:00", "mjpeg": {"age_sec": 0.381, "exists": true, "mtime": "2026-07-02T12:15:44.364354+09:00", "mtime_epoch": 1782962144.3643541, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199963}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:15:44+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1322.727, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 76, "pushed_frames": 76, "read_frames": 75, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.541, "pushed": 2.541, "read": 2.508}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 21.249, "count": 75, "max_ms": 27.414, "p95_ms": 24.2}, "gstreamer_push": {"avg_ms": 0.873, "count": 76, "max_ms": 1.048, "p95_ms": 0.924}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.858, "count": 75, "max_ms": 70.644, "p95_ms": 67.63}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.833, "count": 75, "max_ms": 106.569, "p95_ms": 98.983}, "yolo11_640_fire_11class": {"avg_ms": 49.485, "count": 75, "max_ms": 69.433, "p95_ms": 64.214}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.161, "count": 75, "max_ms": 109.988, "p95_ms": 102.869}}, "truncated_count": 0}, "processing_total": {"avg_ms": 389.925, "count": 76, "max_ms": 453.568, "p95_ms": 436.519}, "read_interval": {"avg_ms": 399.146, "count": 75, "max_ms": 462.616, "p95_ms": 444.497}, "triton_wait": {"avg_ms": 78.851, "count": 75, "max_ms": 110.001, "p95_ms": 106.582}, "visualize": {"avg_ms": 16.783, "count": 75, "max_ms": 21.385, "p95_ms": 18.73}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.907, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 30, "tracking_summary": 676}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5186.695, "limit_mb": 8796093022207.996}, "detail": {"pid": 7255, "restart_count": 29, "start_count": 30}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 22, "h264_decode_error_count": 31, "h264_decode_error_count_per_hour": 18, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:15:44+09:00", "last_error_summary": "[h264 @ 0x55a9b9ca5b80] error while decoding MB 7 41, bytestream -17", "last_exit_at": "2026-07-02T12:15:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T12:15:47+09:00", "pid": 7255, "process": {"exists": true, "fd_count": 5, "hwm_mb": 137.25, "name": "ffmpeg", "pid": 7255, "rss_mb": 137.25, "state": "S (sleeping)", "threads": 106}, "restart_count": 29, "restart_count_per_hour": 22, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 30}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 996.036, "sample_count": 511, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 81128, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:15:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.923, "last_frame_pushed_at": "2026-07-02T12:15:44+09:00", "last_frame_read_age_sec": 5.307, "last_frame_read_at": "2026-07-02T12:15:44+09:00", "mjpeg": {"age_sec": 4.911, "exists": true, "mtime": "2026-07-02T12:15:44.752362+09:00", "mtime_epoch": 1782962144.7523625, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199747}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:15:49+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1322.727, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 64, "pushed_frames": 64, "read_frames": 63, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.158, "pushed": 2.158, "read": 2.125}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 20.923, "count": 64, "max_ms": 27.414, "p95_ms": 22.899}, "gstreamer_push": {"avg_ms": 0.872, "count": 64, "max_ms": 0.935, "p95_ms": 0.924}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.606, "count": 64, "max_ms": 70.644, "p95_ms": 67.63}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 73.896, "count": 64, "max_ms": 104.425, "p95_ms": 98.33}, "yolo11_640_fire_11class": {"avg_ms": 50.191, "count": 64, "max_ms": 69.433, "p95_ms": 64.214}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.782, "count": 64, "max_ms": 109.988, "p95_ms": 102.869}}, "truncated_count": 0}, "processing_total": {"avg_ms": 384.093, "count": 64, "max_ms": 437.679, "p95_ms": 421.257}, "read_interval": {"avg_ms": 393.176, "count": 63, "max_ms": 444.497, "p95_ms": 430.57}, "triton_wait": {"avg_ms": 79.234, "count": 64, "max_ms": 110.001, "p95_ms": 104.436}, "visualize": {"avg_ms": 16.524, "count": 64, "max_ms": 20.695, "p95_ms": 18.435}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.653, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 29, "tracking_summary": 710}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5033.227, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x559ebd5afe40] error while decoding MB 90 25, bytestream -31", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 14, "h264_decode_error_count": 14, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:16:37+09:00", "last_error_summary": "[h264 @ 0x559ebd5afe40] error while decoding MB 90 25, bytestream -31", "last_exit_at": "2026-07-02T11:57:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 1150.694, "last_started_at": "2026-07-02T11:57:26+09:00", "pid": 6114, "process": {"exists": false, "pid": 6114}, "restart_count": 13, "restart_count_per_hour": 6, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 14}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.03, "sample_count": 2387, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 85287, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:16:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.075, "last_frame_pushed_at": "2026-07-02T12:16:37+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-02T12:16:37+09:00", "mjpeg": {"age_sec": 0.056, "exists": true, "mtime": "2026-07-02T12:16:37.533503+09:00", "mtime_epoch": 1782962197.533503, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 247167}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T12:16:37+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1072.504, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 238, "pushed_frames": 238, "read_frames": 238, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.941, "pushed": 7.941, "read": 7.941}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.055, "count": 238, "max_ms": 9.025, "p95_ms": 7.19}, "gstreamer_push": {"avg_ms": 1.031, "count": 238, "max_ms": 1.564, "p95_ms": 1.187}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.311, "count": 238, "max_ms": 82.998, "p95_ms": 58.899}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.052, "count": 238, "max_ms": 120.373, "p95_ms": 85.541}, "yolo11_640_fire_11class": {"avg_ms": 49.222, "count": 238, "max_ms": 76.689, "p95_ms": 60.23}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.929, "count": 238, "max_ms": 119.576, "p95_ms": 94.81}}, "truncated_count": 0}, "processing_total": {"avg_ms": 113.562, "count": 238, "max_ms": 175.171, "p95_ms": 150.483}, "read_interval": {"avg_ms": 126.129, "count": 238, "max_ms": 188.928, "p95_ms": 161.937}, "triton_wait": {"avg_ms": 60.443, "count": 238, "max_ms": 120.384, "p95_ms": 98.918}, "visualize": {"avg_ms": 5.0, "count": 238, "max_ms": 6.565, "p95_ms": 5.71}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.969, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 29, "tracking_summary": 710}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5032.426, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T12:16:37+09:00", "last_error_summary": "[h264 @ 0x559ebd5afe40] error while decoding MB 90 25, bytestream -31", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 14, "h264_decode_error_count": 14, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:16:37+09:00", "last_error_summary": "[h264 @ 0x559ebd5afe40] error while decoding MB 90 25, bytestream -31", "last_exit_at": "2026-07-02T12:16:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 1150.752, "last_started_at": "2026-07-02T11:57:26+09:00", "pid": 6114, "process": {"exists": false, "pid": 6114}, "restart_count": 13, "restart_count_per_hour": 6, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 14}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.03, "sample_count": 2388, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 85287, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:16:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T12:16:37+09:00", "last_frame_read_age_sec": 0.122, "last_frame_read_at": "2026-07-02T12:16:37+09:00", "mjpeg": {"age_sec": 0.115, "exists": true, "mtime": "2026-07-02T12:16:37.533503+09:00", "mtime_epoch": 1782962197.533503, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 247167}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T12:16:37+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1072.504, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 238, "pushed_frames": 238, "read_frames": 237, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.961, "pushed": 7.961, "read": 7.928}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.055, "count": 238, "max_ms": 9.025, "p95_ms": 7.19}, "gstreamer_push": {"avg_ms": 1.031, "count": 238, "max_ms": 1.564, "p95_ms": 1.187}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.311, "count": 238, "max_ms": 82.998, "p95_ms": 58.899}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.052, "count": 238, "max_ms": 120.373, "p95_ms": 85.541}, "yolo11_640_fire_11class": {"avg_ms": 49.222, "count": 238, "max_ms": 76.689, "p95_ms": 60.23}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.929, "count": 238, "max_ms": 119.576, "p95_ms": 94.81}}, "truncated_count": 0}, "processing_total": {"avg_ms": 113.565, "count": 238, "max_ms": 175.171, "p95_ms": 150.483}, "read_interval": {"avg_ms": 126.13, "count": 237, "max_ms": 188.928, "p95_ms": 161.937}, "triton_wait": {"avg_ms": 60.443, "count": 238, "max_ms": 120.384, "p95_ms": 98.918}, "visualize": {"avg_ms": 5.0, "count": 238, "max_ms": 6.565, "p95_ms": 5.71}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.896, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 29, "tracking_summary": 710}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5032.414, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 14, "h264_decode_error_count": 14, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:16:37+09:00", "last_error_summary": "[h264 @ 0x559ebd5afe40] error while decoding MB 90 25, bytestream -31", "last_exit_at": "2026-07-02T12:16:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 1150.76, "last_started_at": "2026-07-02T11:57:26+09:00", "pid": 6114, "process": {"exists": false, "pid": 6114}, "restart_count": 13, "restart_count_per_hour": 6, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 14}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.03, "sample_count": 2388, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 85287, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:16:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-02T12:16:37+09:00", "last_frame_read_age_sec": 0.13, "last_frame_read_at": "2026-07-02T12:16:37+09:00", "mjpeg": {"age_sec": 0.122, "exists": true, "mtime": "2026-07-02T12:16:37.533503+09:00", "mtime_epoch": 1782962197.533503, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 247167}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T12:16:37+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1072.504, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 238, "pushed_frames": 238, "read_frames": 237, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.959, "pushed": 7.959, "read": 7.926}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.055, "count": 238, "max_ms": 9.025, "p95_ms": 7.19}, "gstreamer_push": {"avg_ms": 1.031, "count": 238, "max_ms": 1.564, "p95_ms": 1.187}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.311, "count": 238, "max_ms": 82.998, "p95_ms": 58.899}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.052, "count": 238, "max_ms": 120.373, "p95_ms": 85.541}, "yolo11_640_fire_11class": {"avg_ms": 49.222, "count": 238, "max_ms": 76.689, "p95_ms": 60.23}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.929, "count": 238, "max_ms": 119.576, "p95_ms": 94.81}}, "truncated_count": 0}, "processing_total": {"avg_ms": 113.565, "count": 238, "max_ms": 175.171, "p95_ms": 150.483}, "read_interval": {"avg_ms": 126.13, "count": 237, "max_ms": 188.928, "p95_ms": 161.937}, "triton_wait": {"avg_ms": 60.443, "count": 238, "max_ms": 120.384, "p95_ms": 98.918}, "visualize": {"avg_ms": 5.0, "count": 238, "max_ms": 6.565, "p95_ms": 5.71}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.903, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 29, "tracking_summary": 710}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5062.914, "limit_mb": 8796093022207.996}, "detail": {"pid": 7422, "restart_count": 14, "start_count": 15}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 14, "h264_decode_error_count": 14, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:16:37+09:00", "last_error_summary": "[h264 @ 0x559ebd5afe40] error while decoding MB 90 25, bytestream -31", "last_exit_at": "2026-07-02T12:16:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T12:16:39+09:00", "pid": 7422, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.918, "name": "ffmpeg", "pid": 7422, "rss_mb": 57.918, "state": "S (sleeping)", "threads": 1}, "restart_count": 14, "restart_count_per_hour": 7, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 15}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.03, "sample_count": 2353, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 85287, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:16:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 4.271, "last_frame_pushed_at": "2026-07-02T12:16:37+09:00", "last_frame_read_age_sec": 4.387, "last_frame_read_at": "2026-07-02T12:16:37+09:00", "mjpeg": {"age_sec": 4.256, "exists": true, "mtime": "2026-07-02T12:16:37.657506+09:00", "mtime_epoch": 1782962197.6575058, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 247167}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T12:16:41+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1072.504, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 205, "pushed_frames": 205, "read_frames": 204, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.838, "pushed": 6.838, "read": 6.805}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.047, "count": 204, "max_ms": 9.025, "p95_ms": 7.156}, "gstreamer_push": {"avg_ms": 1.026, "count": 205, "max_ms": 1.564, "p95_ms": 1.18}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.298, "count": 204, "max_ms": 82.998, "p95_ms": 58.899}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.888, "count": 204, "max_ms": 120.373, "p95_ms": 85.413}, "yolo11_640_fire_11class": {"avg_ms": 49.136, "count": 204, "max_ms": 76.689, "p95_ms": 60.23}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.755, "count": 204, "max_ms": 119.282, "p95_ms": 93.827}}, "truncated_count": 0}, "processing_total": {"avg_ms": 113.456, "count": 205, "max_ms": 175.014, "p95_ms": 146.615}, "read_interval": {"avg_ms": 125.979, "count": 204, "max_ms": 188.928, "p95_ms": 159.077}, "triton_wait": {"avg_ms": 60.379, "count": 204, "max_ms": 120.384, "p95_ms": 95.082}, "visualize": {"avg_ms": 5.002, "count": 204, "max_ms": 6.565, "p95_ms": 5.71}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.978, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 36, "tracking_summary": 678}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5061.434, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5646362101c0] error while decoding MB 33 25, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 23, "h264_decode_error_count": 32, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:17:59+09:00", "last_error_summary": "[h264 @ 0x5646362101c0] error while decoding MB 33 25, bytestream -7", "last_exit_at": "2026-07-02T12:15:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.553, "last_started_at": "2026-07-02T12:15:47+09:00", "pid": 7255, "process": {"exists": false, "pid": 7255}, "restart_count": 29, "restart_count_per_hour": 22, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 30}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 761.025, "sample_count": 637, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 81401, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:17:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-02T12:17:59+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-02T12:17:59+09:00", "mjpeg": {"age_sec": 0.04, "exists": true, "mtime": "2026-07-02T12:17:59.175256+09:00", "mtime_epoch": 1782962279.1752565, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200683}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:17:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1322.707, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 69, "pushed_frames": 69, "read_frames": 69, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.323, "pushed": 2.323, "read": 2.323}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 21.793, "count": 68, "max_ms": 26.968, "p95_ms": 26.343}, "gstreamer_push": {"avg_ms": 0.871, "count": 69, "max_ms": 1.068, "p95_ms": 0.946}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.475, "count": 68, "max_ms": 67.607, "p95_ms": 66.632}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 75.911, "count": 68, "max_ms": 103.895, "p95_ms": 93.54}, "yolo11_640_fire_11class": {"avg_ms": 50.962, "count": 68, "max_ms": 67.798, "p95_ms": 65.833}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.39, "count": 68, "max_ms": 107.335, "p95_ms": 95.815}}, "truncated_count": 0}, "processing_total": {"avg_ms": 426.692, "count": 69, "max_ms": 502.396, "p95_ms": 465.261}, "read_interval": {"avg_ms": 435.998, "count": 69, "max_ms": 511.134, "p95_ms": 475.757}, "triton_wait": {"avg_ms": 79.211, "count": 68, "max_ms": 107.347, "p95_ms": 96.237}, "visualize": {"avg_ms": 17.492, "count": 68, "max_ms": 21.71, "p95_ms": 19.19}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.701, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 36, "tracking_summary": 678}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5060.32, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T12:17:59+09:00", "last_error_summary": "[h264 @ 0x5646362101c0] error while decoding MB 33 25, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 23, "h264_decode_error_count": 32, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:17:59+09:00", "last_error_summary": "[h264 @ 0x5646362101c0] error while decoding MB 33 25, bytestream -7", "last_exit_at": "2026-07-02T12:17:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.95, "last_started_at": "2026-07-02T12:15:47+09:00", "pid": 7255, "process": {"exists": false, "pid": 7255}, "restart_count": 29, "restart_count_per_hour": 22, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 30}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 756.94, "sample_count": 637, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 81402, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:17:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T12:17:59+09:00", "last_frame_read_age_sec": 0.447, "last_frame_read_at": "2026-07-02T12:17:59+09:00", "mjpeg": {"age_sec": 0.437, "exists": true, "mtime": "2026-07-02T12:17:59.175256+09:00", "mtime_epoch": 1782962279.1752565, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200683}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:17:59+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1322.707, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 69, "pushed_frames": 69, "read_frames": 68, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.326, "pushed": 2.326, "read": 2.293}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 21.778, "count": 69, "max_ms": 26.968, "p95_ms": 26.343}, "gstreamer_push": {"avg_ms": 0.87, "count": 69, "max_ms": 1.068, "p95_ms": 0.946}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.567, "count": 69, "max_ms": 67.607, "p95_ms": 66.632}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.047, "count": 69, "max_ms": 103.895, "p95_ms": 93.54}, "yolo11_640_fire_11class": {"avg_ms": 51.063, "count": 69, "max_ms": 67.798, "p95_ms": 65.833}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.641, "count": 69, "max_ms": 107.335, "p95_ms": 95.815}}, "truncated_count": 0}, "processing_total": {"avg_ms": 426.84, "count": 69, "max_ms": 502.396, "p95_ms": 465.261}, "read_interval": {"avg_ms": 435.914, "count": 68, "max_ms": 511.134, "p95_ms": 475.757}, "triton_wait": {"avg_ms": 79.436, "count": 69, "max_ms": 107.347, "p95_ms": 96.237}, "visualize": {"avg_ms": 17.478, "count": 69, "max_ms": 21.71, "p95_ms": 19.19}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.659, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 36, "tracking_summary": 678}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5060.312, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 23, "h264_decode_error_count": 32, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:17:59+09:00", "last_error_summary": "[h264 @ 0x5646362101c0] error while decoding MB 33 25, bytestream -7", "last_exit_at": "2026-07-02T12:17:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.953, "last_started_at": "2026-07-02T12:15:47+09:00", "pid": 7255, "process": {"exists": false, "pid": 7255}, "restart_count": 29, "restart_count_per_hour": 22, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 30}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 756.94, "sample_count": 637, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 81402, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:17:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T12:17:59+09:00", "last_frame_read_age_sec": 0.45, "last_frame_read_at": "2026-07-02T12:17:59+09:00", "mjpeg": {"age_sec": 0.44, "exists": true, "mtime": "2026-07-02T12:17:59.175256+09:00", "mtime_epoch": 1782962279.1752565, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200683}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:17:59+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1322.707, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 69, "pushed_frames": 69, "read_frames": 68, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.326, "pushed": 2.326, "read": 2.292}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 21.778, "count": 69, "max_ms": 26.968, "p95_ms": 26.343}, "gstreamer_push": {"avg_ms": 0.87, "count": 69, "max_ms": 1.068, "p95_ms": 0.946}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.567, "count": 69, "max_ms": 67.607, "p95_ms": 66.632}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 76.047, "count": 69, "max_ms": 103.895, "p95_ms": 93.54}, "yolo11_640_fire_11class": {"avg_ms": 51.063, "count": 69, "max_ms": 67.798, "p95_ms": 65.833}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.641, "count": 69, "max_ms": 107.335, "p95_ms": 95.815}}, "truncated_count": 0}, "processing_total": {"avg_ms": 426.84, "count": 69, "max_ms": 502.396, "p95_ms": 465.261}, "read_interval": {"avg_ms": 435.914, "count": 68, "max_ms": 511.134, "p95_ms": 475.757}, "triton_wait": {"avg_ms": 79.436, "count": 69, "max_ms": 107.347, "p95_ms": 96.237}, "visualize": {"avg_ms": 17.478, "count": 69, "max_ms": 21.71, "p95_ms": 19.19}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.662, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 36, "tracking_summary": 678}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5171.387, "limit_mb": 8796093022207.996}, "detail": {"pid": 7574, "restart_count": 30, "start_count": 31}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 23, "h264_decode_error_count": 32, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:17:59+09:00", "last_error_summary": "[h264 @ 0x5646362101c0] error while decoding MB 33 25, bytestream -7", "last_exit_at": "2026-07-02T12:18:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T12:18:02+09:00", "pid": 7574, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.242, "name": "ffmpeg", "pid": 7574, "rss_mb": 139.242, "state": "S (sleeping)", "threads": 106}, "restart_count": 30, "restart_count_per_hour": 23, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 31}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 749.273, "sample_count": 632, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 81402, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:17:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.69, "last_frame_pushed_at": "2026-07-02T12:17:59+09:00", "last_frame_read_age_sec": 5.134, "last_frame_read_at": "2026-07-02T12:17:59+09:00", "mjpeg": {"age_sec": 4.672, "exists": true, "mtime": "2026-07-02T12:17:59.627266+09:00", "mtime_epoch": 1782962279.627266, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 195595}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:18:04+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1322.707, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 59, "pushed_frames": 59, "read_frames": 58, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.978, "pushed": 1.978, "read": 1.944}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 21.323, "count": 58, "max_ms": 26.343, "p95_ms": 24.545}, "gstreamer_push": {"avg_ms": 0.871, "count": 59, "max_ms": 1.068, "p95_ms": 0.959}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.724, "count": 58, "max_ms": 67.607, "p95_ms": 66.767}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 74.989, "count": 58, "max_ms": 101.449, "p95_ms": 93.205}, "yolo11_640_fire_11class": {"avg_ms": 51.048, "count": 58, "max_ms": 67.798, "p95_ms": 66.619}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 77.956, "count": 58, "max_ms": 107.335, "p95_ms": 96.225}}, "truncated_count": 0}, "processing_total": {"avg_ms": 424.638, "count": 59, "max_ms": 502.396, "p95_ms": 465.261}, "read_interval": {"avg_ms": 433.571, "count": 58, "max_ms": 511.134, "p95_ms": 475.757}, "triton_wait": {"avg_ms": 78.879, "count": 58, "max_ms": 107.347, "p95_ms": 96.237}, "visualize": {"avg_ms": 17.546, "count": 58, "max_ms": 21.71, "p95_ms": 19.252}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.83, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 36, "tracking_summary": 681}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5055.816, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T12:17:59+09:00", "last_error_summary": "[h264 @ 0x5646362101c0] error while decoding MB 33 25, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 23, "h264_decode_error_count": 32, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:17:59+09:00", "last_error_summary": "[h264 @ 0x5646362101c0] error while decoding MB 33 25, bytestream -7", "last_exit_at": "2026-07-02T12:20:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 118.753, "last_started_at": "2026-07-02T12:18:02+09:00", "pid": 7574, "process": {"exists": false, "pid": 7574}, "restart_count": 30, "restart_count_per_hour": 23, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 31}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 531.844, "sample_count": 807, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 81827, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:20:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T12:20:01+09:00", "last_frame_read_age_sec": 0.287, "last_frame_read_at": "2026-07-02T12:20:00+09:00", "mjpeg": {"age_sec": 0.281, "exists": true, "mtime": "2026-07-02T12:20:00.769847+09:00", "mtime_epoch": 1782962400.7698474, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192420}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:20:01+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1322.965, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 110, "pushed_frames": 110, "read_frames": 109, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.693, "pushed": 3.693, "read": 3.66}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 13.156, "count": 110, "max_ms": 17.467, "p95_ms": 15.418}, "gstreamer_push": {"avg_ms": 0.868, "count": 110, "max_ms": 1.05, "p95_ms": 0.928}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.64, "count": 110, "max_ms": 72.243, "p95_ms": 65.214}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.38, "count": 110, "max_ms": 111.776, "p95_ms": 94.984}, "yolo11_640_fire_11class": {"avg_ms": 49.951, "count": 110, "max_ms": 73.859, "p95_ms": 67.235}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.492, "count": 110, "max_ms": 101.901, "p95_ms": 90.145}}, "truncated_count": 0}, "processing_total": {"avg_ms": 263.633, "count": 110, "max_ms": 310.705, "p95_ms": 301.932}, "read_interval": {"avg_ms": 273.435, "count": 109, "max_ms": 323.066, "p95_ms": 311.999}, "triton_wait": {"avg_ms": 72.166, "count": 110, "max_ms": 111.788, "p95_ms": 100.288}, "visualize": {"avg_ms": 12.01, "count": 110, "max_ms": 14.596, "p95_ms": 13.492}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.784, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 36, "tracking_summary": 681}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5056.809, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 23, "h264_decode_error_count": 32, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:17:59+09:00", "last_error_summary": "[h264 @ 0x5646362101c0] error while decoding MB 33 25, bytestream -7", "last_exit_at": "2026-07-02T12:20:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 118.756, "last_started_at": "2026-07-02T12:18:02+09:00", "pid": 7574, "process": {"exists": false, "pid": 7574}, "restart_count": 30, "restart_count_per_hour": 23, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 31}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 531.844, "sample_count": 807, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 81827, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:20:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T12:20:01+09:00", "last_frame_read_age_sec": 0.29, "last_frame_read_at": "2026-07-02T12:20:00+09:00", "mjpeg": {"age_sec": 0.285, "exists": true, "mtime": "2026-07-02T12:20:00.769847+09:00", "mtime_epoch": 1782962400.7698474, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192420}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:20:01+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1322.965, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 110, "pushed_frames": 110, "read_frames": 109, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.693, "pushed": 3.693, "read": 3.659}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 13.156, "count": 110, "max_ms": 17.467, "p95_ms": 15.418}, "gstreamer_push": {"avg_ms": 0.868, "count": 110, "max_ms": 1.05, "p95_ms": 0.928}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.64, "count": 110, "max_ms": 72.243, "p95_ms": 65.214}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 70.38, "count": 110, "max_ms": 111.776, "p95_ms": 94.984}, "yolo11_640_fire_11class": {"avg_ms": 49.951, "count": 110, "max_ms": 73.859, "p95_ms": 67.235}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.492, "count": 110, "max_ms": 101.901, "p95_ms": 90.145}}, "truncated_count": 0}, "processing_total": {"avg_ms": 263.633, "count": 110, "max_ms": 310.705, "p95_ms": 301.932}, "read_interval": {"avg_ms": 273.435, "count": 109, "max_ms": 323.066, "p95_ms": 311.999}, "triton_wait": {"avg_ms": 72.166, "count": 110, "max_ms": 111.788, "p95_ms": 100.288}, "visualize": {"avg_ms": 12.01, "count": 110, "max_ms": 14.596, "p95_ms": 13.492}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.787, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 36, "tracking_summary": 681}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5172.895, "limit_mb": 8796093022207.996}, "detail": {"pid": 7681, "restart_count": 31, "start_count": 32}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 23, "h264_decode_error_count": 32, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:17:59+09:00", "last_error_summary": "[h264 @ 0x5646362101c0] error while decoding MB 33 25, bytestream -7", "last_exit_at": "2026-07-02T12:20:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T12:20:04+09:00", "pid": 7681, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.707, "name": "ffmpeg", "pid": 7681, "rss_mb": 141.707, "state": "S (sleeping)", "threads": 106}, "restart_count": 31, "restart_count_per_hour": 24, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 32}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 531.844, "sample_count": 795, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 81827, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:20:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.996, "last_frame_pushed_at": "2026-07-02T12:20:01+09:00", "last_frame_read_age_sec": 5.279, "last_frame_read_at": "2026-07-02T12:20:00+09:00", "mjpeg": {"age_sec": 4.977, "exists": true, "mtime": "2026-07-02T12:20:01.065854+09:00", "mtime_epoch": 1782962401.0658536, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192962}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:20:06+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1322.965, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 93, "pushed_frames": 93, "read_frames": 92, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.109, "pushed": 3.109, "read": 3.076}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 13.092, "count": 92, "max_ms": 17.467, "p95_ms": 15.191}, "gstreamer_push": {"avg_ms": 0.867, "count": 93, "max_ms": 1.05, "p95_ms": 0.928}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.404, "count": 92, "max_ms": 72.243, "p95_ms": 63.829}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 69.318, "count": 92, "max_ms": 111.776, "p95_ms": 91.689}, "yolo11_640_fire_11class": {"avg_ms": 49.465, "count": 92, "max_ms": 73.859, "p95_ms": 66.408}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.059, "count": 92, "max_ms": 99.922, "p95_ms": 88.737}}, "truncated_count": 0}, "processing_total": {"avg_ms": 261.428, "count": 93, "max_ms": 310.705, "p95_ms": 293.378}, "read_interval": {"avg_ms": 271.144, "count": 92, "max_ms": 323.066, "p95_ms": 303.506}, "triton_wait": {"avg_ms": 71.322, "count": 92, "max_ms": 111.788, "p95_ms": 91.726}, "visualize": {"avg_ms": 11.915, "count": 92, "max_ms": 14.596, "p95_ms": 13.541}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.914, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 40, "tracking_summary": 685}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5073.504, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x555bec0b9500] error while decoding MB 45 39, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 24, "h264_decode_error_count": 33, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:21:53+09:00", "last_error_summary": "[h264 @ 0x555bec0b9500] error while decoding MB 45 39, bytestream -7", "last_exit_at": "2026-07-02T12:20:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.557, "last_started_at": "2026-07-02T12:20:04+09:00", "pid": 7681, "process": {"exists": false, "pid": 7681}, "restart_count": 31, "restart_count_per_hour": 24, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 32}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 465.261, "sample_count": 981, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 82241, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:21:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.06, "last_frame_pushed_at": "2026-07-02T12:21:53+09:00", "last_frame_read_age_sec": 0.052, "last_frame_read_at": "2026-07-02T12:21:53+09:00", "mjpeg": {"age_sec": 0.039, "exists": true, "mtime": "2026-07-02T12:21:53.560233+09:00", "mtime_epoch": 1782962513.5602329, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191358}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:21:53+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1322.195, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 122, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.071, "pushed": 4.071, "read": 4.071}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 11.947, "count": 121, "max_ms": 17.952, "p95_ms": 13.932}, "gstreamer_push": {"avg_ms": 0.873, "count": 122, "max_ms": 1.041, "p95_ms": 0.924}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.601, "count": 121, "max_ms": 68.906, "p95_ms": 62.865}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.205, "count": 121, "max_ms": 111.112, "p95_ms": 90.596}, "yolo11_640_fire_11class": {"avg_ms": 48.666, "count": 121, "max_ms": 72.787, "p95_ms": 64.858}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.711, "count": 121, "max_ms": 98.285, "p95_ms": 89.9}}, "truncated_count": 0}, "processing_total": {"avg_ms": 238.076, "count": 122, "max_ms": 284.508, "p95_ms": 275.354}, "read_interval": {"avg_ms": 247.187, "count": 122, "max_ms": 293.548, "p95_ms": 284.788}, "triton_wait": {"avg_ms": 68.121, "count": 121, "max_ms": 111.124, "p95_ms": 91.982}, "visualize": {"avg_ms": 10.67, "count": 121, "max_ms": 15.275, "p95_ms": 12.43}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.971, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 40, "tracking_summary": 685}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5075.492, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T12:21:53+09:00", "last_error_summary": "[h264 @ 0x555bec0b9500] error while decoding MB 45 39, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 24, "h264_decode_error_count": 33, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:21:53+09:00", "last_error_summary": "[h264 @ 0x555bec0b9500] error while decoding MB 45 39, bytestream -7", "last_exit_at": "2026-07-02T12:21:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.75, "last_started_at": "2026-07-02T12:20:04+09:00", "pid": 7681, "process": {"exists": false, "pid": 7681}, "restart_count": 31, "restart_count_per_hour": 24, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 32}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 465.261, "sample_count": 982, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 82242, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:21:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T12:21:53+09:00", "last_frame_read_age_sec": 0.245, "last_frame_read_at": "2026-07-02T12:21:53+09:00", "mjpeg": {"age_sec": 0.232, "exists": true, "mtime": "2026-07-02T12:21:53.560233+09:00", "mtime_epoch": 1782962513.5602329, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191358}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:21:53+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1322.195, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 121, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.08, "pushed": 4.08, "read": 4.047}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 11.935, "count": 121, "max_ms": 17.952, "p95_ms": 13.932}, "gstreamer_push": {"avg_ms": 0.873, "count": 122, "max_ms": 1.041, "p95_ms": 0.924}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.622, "count": 121, "max_ms": 68.906, "p95_ms": 62.865}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.206, "count": 121, "max_ms": 111.112, "p95_ms": 90.596}, "yolo11_640_fire_11class": {"avg_ms": 48.687, "count": 121, "max_ms": 72.787, "p95_ms": 64.858}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.716, "count": 121, "max_ms": 98.285, "p95_ms": 89.9}}, "truncated_count": 0}, "processing_total": {"avg_ms": 238.079, "count": 122, "max_ms": 284.508, "p95_ms": 275.354}, "read_interval": {"avg_ms": 247.176, "count": 121, "max_ms": 293.548, "p95_ms": 284.788}, "triton_wait": {"avg_ms": 68.122, "count": 121, "max_ms": 111.124, "p95_ms": 91.982}, "visualize": {"avg_ms": 10.656, "count": 121, "max_ms": 15.275, "p95_ms": 12.43}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.899, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 40, "tracking_summary": 685}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5075.484, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 24, "h264_decode_error_count": 33, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:21:53+09:00", "last_error_summary": "[h264 @ 0x555bec0b9500] error while decoding MB 45 39, bytestream -7", "last_exit_at": "2026-07-02T12:21:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 109.753, "last_started_at": "2026-07-02T12:20:04+09:00", "pid": 7681, "process": {"exists": false, "pid": 7681}, "restart_count": 31, "restart_count_per_hour": 24, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 32}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 465.261, "sample_count": 982, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 82242, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:21:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-02T12:21:53+09:00", "last_frame_read_age_sec": 0.248, "last_frame_read_at": "2026-07-02T12:21:53+09:00", "mjpeg": {"age_sec": 0.235, "exists": true, "mtime": "2026-07-02T12:21:53.560233+09:00", "mtime_epoch": 1782962513.5602329, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191358}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:21:53+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1322.195, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 122, "pushed_frames": 122, "read_frames": 121, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.08, "pushed": 4.08, "read": 4.046}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 11.935, "count": 121, "max_ms": 17.952, "p95_ms": 13.932}, "gstreamer_push": {"avg_ms": 0.873, "count": 122, "max_ms": 1.041, "p95_ms": 0.924}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.622, "count": 121, "max_ms": 68.906, "p95_ms": 62.865}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.206, "count": 121, "max_ms": 111.112, "p95_ms": 90.596}, "yolo11_640_fire_11class": {"avg_ms": 48.687, "count": 121, "max_ms": 72.787, "p95_ms": 64.858}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.716, "count": 121, "max_ms": 98.285, "p95_ms": 89.9}}, "truncated_count": 0}, "processing_total": {"avg_ms": 238.079, "count": 122, "max_ms": 284.508, "p95_ms": 275.354}, "read_interval": {"avg_ms": 247.176, "count": 121, "max_ms": 293.548, "p95_ms": 284.788}, "triton_wait": {"avg_ms": 68.122, "count": 121, "max_ms": 111.124, "p95_ms": 91.982}, "visualize": {"avg_ms": 10.656, "count": 121, "max_ms": 15.275, "p95_ms": 12.43}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.903, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 40, "tracking_summary": 685}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5178.668, "limit_mb": 8796093022207.996}, "detail": {"pid": 7848, "restart_count": 32, "start_count": 33}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 24, "h264_decode_error_count": 33, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:21:53+09:00", "last_error_summary": "[h264 @ 0x555bec0b9500] error while decoding MB 45 39, bytestream -7", "last_exit_at": "2026-07-02T12:21:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T12:21:56+09:00", "pid": 7848, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.285, "name": "ffmpeg", "pid": 7848, "rss_mb": 136.285, "state": "S (sleeping)", "threads": 106}, "restart_count": 32, "restart_count_per_hour": 25, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 33}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 456.222, "sample_count": 973, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 82242, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:21:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.676, "last_frame_pushed_at": "2026-07-02T12:21:53+09:00", "last_frame_read_age_sec": 4.917, "last_frame_read_at": "2026-07-02T12:21:53+09:00", "mjpeg": {"age_sec": 4.664, "exists": true, "mtime": "2026-07-02T12:21:53.800238+09:00", "mtime_epoch": 1782962513.800238, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191464}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:21:58+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1322.195, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 103, "pushed_frames": 103, "read_frames": 102, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.461, "pushed": 3.461, "read": 3.427}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 11.903, "count": 103, "max_ms": 17.952, "p95_ms": 13.881}, "gstreamer_push": {"avg_ms": 0.874, "count": 103, "max_ms": 1.041, "p95_ms": 0.927}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.383, "count": 103, "max_ms": 68.906, "p95_ms": 62.765}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.212, "count": 103, "max_ms": 111.112, "p95_ms": 91.346}, "yolo11_640_fire_11class": {"avg_ms": 48.46, "count": 103, "max_ms": 72.787, "p95_ms": 64.408}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.946, "count": 103, "max_ms": 98.285, "p95_ms": 88.663}}, "truncated_count": 0}, "processing_total": {"avg_ms": 236.755, "count": 103, "max_ms": 284.508, "p95_ms": 275.354}, "read_interval": {"avg_ms": 245.951, "count": 102, "max_ms": 293.548, "p95_ms": 284.788}, "triton_wait": {"avg_ms": 67.621, "count": 103, "max_ms": 111.124, "p95_ms": 91.982}, "visualize": {"avg_ms": 10.607, "count": 103, "max_ms": 15.275, "p95_ms": 11.784}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.762, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 40, "tracking_summary": 691}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5074.062, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x56155d706e40] error while decoding MB 28 13, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 25, "h264_decode_error_count": 34, "h264_decode_error_count_per_hour": 21, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:24:08+09:00", "last_error_summary": "[h264 @ 0x56155d706e40] error while decoding MB 28 13, bytestream -5", "last_exit_at": "2026-07-02T12:21:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.224, "last_started_at": "2026-07-02T12:21:56+09:00", "pid": 7848, "process": {"exists": false, "pid": 7848}, "restart_count": 32, "restart_count_per_hour": 25, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 33}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 300.212, "sample_count": 1394, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 83034, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:24:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.065, "last_frame_pushed_at": "2026-07-02T12:24:08+09:00", "last_frame_read_age_sec": 0.058, "last_frame_read_at": "2026-07-02T12:24:08+09:00", "mjpeg": {"age_sec": 0.048, "exists": true, "mtime": "2026-07-02T12:24:08.639072+09:00", "mtime_epoch": 1782962648.6390722, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191043}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:24:08+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1322.711, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 170, "pushed_frames": 170, "read_frames": 170, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.686, "pushed": 5.686, "read": 5.686}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.004, "count": 170, "max_ms": 12.021, "p95_ms": 9.145}, "gstreamer_push": {"avg_ms": 0.953, "count": 170, "max_ms": 1.154, "p95_ms": 1.097}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.565, "count": 170, "max_ms": 73.573, "p95_ms": 69.067}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.399, "count": 170, "max_ms": 110.184, "p95_ms": 97.749}, "yolo11_640_fire_11class": {"avg_ms": 54.108, "count": 170, "max_ms": 80.403, "p95_ms": 72.465}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.789, "count": 170, "max_ms": 95.615, "p95_ms": 89.529}}, "truncated_count": 0}, "processing_total": {"avg_ms": 166.414, "count": 170, "max_ms": 217.685, "p95_ms": 199.852}, "read_interval": {"avg_ms": 176.645, "count": 170, "max_ms": 227.675, "p95_ms": 209.459}, "triton_wait": {"avg_ms": 71.254, "count": 170, "max_ms": 110.197, "p95_ms": 97.761}, "visualize": {"avg_ms": 7.094, "count": 170, "max_ms": 8.882, "p95_ms": 8.06}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.899, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 40, "tracking_summary": 691}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5076.992, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T12:24:08+09:00", "last_error_summary": "[h264 @ 0x56155d706e40] error while decoding MB 28 13, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 25, "h264_decode_error_count": 34, "h264_decode_error_count_per_hour": 21, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:24:08+09:00", "last_error_summary": "[h264 @ 0x56155d706e40] error while decoding MB 28 13, bytestream -5", "last_exit_at": "2026-07-02T12:24:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.38, "last_started_at": "2026-07-02T12:21:56+09:00", "pid": 7848, "process": {"exists": false, "pid": 7848}, "restart_count": 32, "restart_count_per_hour": 25, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 33}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 300.212, "sample_count": 1394, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 83035, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:24:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T12:24:08+09:00", "last_frame_read_age_sec": 0.213, "last_frame_read_at": "2026-07-02T12:24:08+09:00", "mjpeg": {"age_sec": 0.203, "exists": true, "mtime": "2026-07-02T12:24:08.639072+09:00", "mtime_epoch": 1782962648.6390722, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191043}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:24:08+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1322.711, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 170, "pushed_frames": 170, "read_frames": 169, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.689, "pushed": 5.689, "read": 5.656}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.007, "count": 170, "max_ms": 12.021, "p95_ms": 9.145}, "gstreamer_push": {"avg_ms": 0.955, "count": 170, "max_ms": 1.154, "p95_ms": 1.107}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.604, "count": 170, "max_ms": 73.573, "p95_ms": 69.067}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.705, "count": 170, "max_ms": 110.184, "p95_ms": 98.689}, "yolo11_640_fire_11class": {"avg_ms": 54.083, "count": 170, "max_ms": 80.403, "p95_ms": 72.465}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.805, "count": 170, "max_ms": 95.615, "p95_ms": 90.305}}, "truncated_count": 0}, "processing_total": {"avg_ms": 166.523, "count": 170, "max_ms": 217.685, "p95_ms": 200.617}, "read_interval": {"avg_ms": 176.514, "count": 169, "max_ms": 227.675, "p95_ms": 209.459}, "triton_wait": {"avg_ms": 71.344, "count": 170, "max_ms": 110.197, "p95_ms": 98.701}, "visualize": {"avg_ms": 7.096, "count": 170, "max_ms": 8.882, "p95_ms": 8.06}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.88, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 40, "tracking_summary": 691}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5076.984, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 25, "h264_decode_error_count": 34, "h264_decode_error_count_per_hour": 21, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:24:08+09:00", "last_error_summary": "[h264 @ 0x56155d706e40] error while decoding MB 28 13, bytestream -5", "last_exit_at": "2026-07-02T12:24:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.383, "last_started_at": "2026-07-02T12:21:56+09:00", "pid": 7848, "process": {"exists": false, "pid": 7848}, "restart_count": 32, "restart_count_per_hour": 25, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 33}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 300.212, "sample_count": 1394, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 83035, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:24:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-02T12:24:08+09:00", "last_frame_read_age_sec": 0.217, "last_frame_read_at": "2026-07-02T12:24:08+09:00", "mjpeg": {"age_sec": 0.207, "exists": true, "mtime": "2026-07-02T12:24:08.639072+09:00", "mtime_epoch": 1782962648.6390722, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191043}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:24:08+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1322.711, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 170, "pushed_frames": 170, "read_frames": 169, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.689, "pushed": 5.689, "read": 5.655}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.007, "count": 170, "max_ms": 12.021, "p95_ms": 9.145}, "gstreamer_push": {"avg_ms": 0.955, "count": 170, "max_ms": 1.154, "p95_ms": 1.107}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.604, "count": 170, "max_ms": 73.573, "p95_ms": 69.067}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.705, "count": 170, "max_ms": 110.184, "p95_ms": 98.689}, "yolo11_640_fire_11class": {"avg_ms": 54.083, "count": 170, "max_ms": 80.403, "p95_ms": 72.465}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.805, "count": 170, "max_ms": 95.615, "p95_ms": 90.305}}, "truncated_count": 0}, "processing_total": {"avg_ms": 166.523, "count": 170, "max_ms": 217.685, "p95_ms": 200.617}, "read_interval": {"avg_ms": 176.514, "count": 169, "max_ms": 227.675, "p95_ms": 209.459}, "triton_wait": {"avg_ms": 71.344, "count": 170, "max_ms": 110.197, "p95_ms": 98.701}, "visualize": {"avg_ms": 7.096, "count": 170, "max_ms": 8.882, "p95_ms": 8.06}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.884, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 40, "tracking_summary": 691}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5184.809, "limit_mb": 8796093022207.996}, "detail": {"pid": 7955, "restart_count": 33, "start_count": 34}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 25, "h264_decode_error_count": 34, "h264_decode_error_count_per_hour": 21, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:24:08+09:00", "last_error_summary": "[h264 @ 0x56155d706e40] error while decoding MB 28 13, bytestream -5", "last_exit_at": "2026-07-02T12:24:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T12:24:10+09:00", "pid": 7955, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.555, "name": "ffmpeg", "pid": 7955, "rss_mb": 138.555, "state": "S (sleeping)", "threads": 106}, "restart_count": 33, "restart_count_per_hour": 26, "rtsp_timeout_count": 6, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 34}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 300.212, "sample_count": 1380, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 83035, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:24:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.068, "last_frame_pushed_at": "2026-07-02T12:24:08+09:00", "last_frame_read_age_sec": 4.277, "last_frame_read_at": "2026-07-02T12:24:08+09:00", "mjpeg": {"age_sec": 4.052, "exists": true, "mtime": "2026-07-02T12:24:08.855077+09:00", "mtime_epoch": 1782962648.8550768, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191383}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T12:24:12+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1322.711, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 148, "pushed_frames": 148, "read_frames": 147, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.96, "pushed": 4.96, "read": 4.927}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.934, "count": 148, "max_ms": 9.823, "p95_ms": 8.995}, "gstreamer_push": {"avg_ms": 0.958, "count": 148, "max_ms": 1.154, "p95_ms": 1.107}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.661, "count": 148, "max_ms": 73.125, "p95_ms": 68.953}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 68.597, "count": 148, "max_ms": 110.184, "p95_ms": 98.689}, "yolo11_640_fire_11class": {"avg_ms": 54.186, "count": 148, "max_ms": 80.403, "p95_ms": 72.465}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.947, "count": 148, "max_ms": 95.615, "p95_ms": 89.217}}, "truncated_count": 0}, "processing_total": {"avg_ms": 165.206, "count": 148, "max_ms": 217.685, "p95_ms": 200.617}, "read_interval": {"avg_ms": 175.114, "count": 147, "max_ms": 227.675, "p95_ms": 206.986}, "triton_wait": {"avg_ms": 71.444, "count": 148, "max_ms": 110.197, "p95_ms": 98.701}, "visualize": {"avg_ms": 7.022, "count": 148, "max_ms": 8.271, "p95_ms": 7.99}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.838, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 29, "tracking_summary": 810}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5057.594, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x562f20eabf80] error while decoding MB 62 30, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 15, "h264_decode_error_count": 15, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:41:47+09:00", "last_error_summary": "[h264 @ 0x562f20eabf80] error while decoding MB 62 30, bytestream -9", "last_exit_at": "2026-07-02T12:16:39+09:00", "last_returncode": 0, "last_stable_duration_sec": 1507.517, "last_started_at": "2026-07-02T12:16:39+09:00", "pid": 7422, "process": {"exists": false, "pid": 7422}, "restart_count": 14, "restart_count_per_hour": 4, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 15}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.349, "sample_count": 2299, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 97205, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:41:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.076, "last_frame_pushed_at": "2026-07-02T12:41:47+09:00", "last_frame_read_age_sec": 0.066, "last_frame_read_at": "2026-07-02T12:41:47+09:00", "mjpeg": {"age_sec": 0.057, "exists": true, "mtime": "2026-07-02T12:41:47.373049+09:00", "mtime_epoch": 1782963707.3730485, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 234832}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T12:41:47+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1077.918, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 228, "pushed_frames": 228, "read_frames": 228, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.614, "pushed": 7.614, "read": 7.614}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.027, "count": 228, "max_ms": 8.153, "p95_ms": 6.878}, "gstreamer_push": {"avg_ms": 1.118, "count": 228, "max_ms": 1.535, "p95_ms": 1.231}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.807, "count": 228, "max_ms": 80.759, "p95_ms": 72.057}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.651, "count": 228, "max_ms": 109.361, "p95_ms": 90.737}, "yolo11_640_fire_11class": {"avg_ms": 56.768, "count": 228, "max_ms": 84.111, "p95_ms": 73.869}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.616, "count": 228, "max_ms": 100.419, "p95_ms": 91.664}}, "truncated_count": 0}, "processing_total": {"avg_ms": 118.837, "count": 228, "max_ms": 162.518, "p95_ms": 148.262}, "read_interval": {"avg_ms": 131.631, "count": 228, "max_ms": 177.378, "p95_ms": 162.913}, "triton_wait": {"avg_ms": 64.854, "count": 228, "max_ms": 109.374, "p95_ms": 93.32}, "visualize": {"avg_ms": 5.162, "count": 228, "max_ms": 6.838, "p95_ms": 5.753}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.944, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 29, "tracking_summary": 810}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5057.152, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T12:41:47+09:00", "last_error_summary": "[h264 @ 0x562f20eabf80] error while decoding MB 62 30, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 15, "h264_decode_error_count": 15, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:41:47+09:00", "last_error_summary": "[h264 @ 0x562f20eabf80] error while decoding MB 62 30, bytestream -9", "last_exit_at": "2026-07-02T12:41:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 1507.582, "last_started_at": "2026-07-02T12:16:39+09:00", "pid": 7422, "process": {"exists": false, "pid": 7422}, "restart_count": 14, "restart_count_per_hour": 4, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 15}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.349, "sample_count": 2299, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 97206, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:41:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T12:41:47+09:00", "last_frame_read_age_sec": 0.131, "last_frame_read_at": "2026-07-02T12:41:47+09:00", "mjpeg": {"age_sec": 0.122, "exists": true, "mtime": "2026-07-02T12:41:47.373049+09:00", "mtime_epoch": 1782963707.3730485, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 234832}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T12:41:47+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1077.918, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 228, "pushed_frames": 228, "read_frames": 228, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.601, "pushed": 7.601, "read": 7.601}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.028, "count": 228, "max_ms": 8.153, "p95_ms": 6.878}, "gstreamer_push": {"avg_ms": 1.118, "count": 228, "max_ms": 1.535, "p95_ms": 1.231}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.736, "count": 228, "max_ms": 80.759, "p95_ms": 72.057}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.701, "count": 228, "max_ms": 109.361, "p95_ms": 90.737}, "yolo11_640_fire_11class": {"avg_ms": 56.711, "count": 228, "max_ms": 84.111, "p95_ms": 73.869}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.52, "count": 228, "max_ms": 100.419, "p95_ms": 91.664}}, "truncated_count": 0}, "processing_total": {"avg_ms": 118.79, "count": 228, "max_ms": 162.518, "p95_ms": 148.262}, "read_interval": {"avg_ms": 131.631, "count": 228, "max_ms": 177.378, "p95_ms": 162.913}, "triton_wait": {"avg_ms": 64.775, "count": 228, "max_ms": 109.374, "p95_ms": 93.32}, "visualize": {"avg_ms": 5.188, "count": 228, "max_ms": 10.978, "p95_ms": 5.789}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 29, "tracking_summary": 810}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5057.262, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 15, "h264_decode_error_count": 15, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:41:47+09:00", "last_error_summary": "[h264 @ 0x562f20eabf80] error while decoding MB 62 30, bytestream -9", "last_exit_at": "2026-07-02T12:41:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 1507.588, "last_started_at": "2026-07-02T12:16:39+09:00", "pid": 7422, "process": {"exists": false, "pid": 7422}, "restart_count": 14, "restart_count_per_hour": 4, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 15}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.349, "sample_count": 2299, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 97206, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:41:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-02T12:41:47+09:00", "last_frame_read_age_sec": 0.137, "last_frame_read_at": "2026-07-02T12:41:47+09:00", "mjpeg": {"age_sec": 0.128, "exists": true, "mtime": "2026-07-02T12:41:47.373049+09:00", "mtime_epoch": 1782963707.3730485, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 234832}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T12:41:47+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1077.918, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 228, "pushed_frames": 228, "read_frames": 227, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.629, "pushed": 7.629, "read": 7.596}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.028, "count": 228, "max_ms": 8.153, "p95_ms": 6.878}, "gstreamer_push": {"avg_ms": 1.118, "count": 228, "max_ms": 1.535, "p95_ms": 1.231}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.736, "count": 228, "max_ms": 80.759, "p95_ms": 72.057}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.701, "count": 228, "max_ms": 109.361, "p95_ms": 90.737}, "yolo11_640_fire_11class": {"avg_ms": 56.711, "count": 228, "max_ms": 84.111, "p95_ms": 73.869}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.52, "count": 228, "max_ms": 100.419, "p95_ms": 91.664}}, "truncated_count": 0}, "processing_total": {"avg_ms": 118.79, "count": 228, "max_ms": 162.518, "p95_ms": 148.262}, "read_interval": {"avg_ms": 131.574, "count": 227, "max_ms": 177.378, "p95_ms": 162.913}, "triton_wait": {"avg_ms": 64.775, "count": 228, "max_ms": 109.374, "p95_ms": 93.32}, "visualize": {"avg_ms": 5.188, "count": 228, "max_ms": 10.978, "p95_ms": 5.789}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.885, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 29, "tracking_summary": 810}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5074.508, "limit_mb": 8796093022207.996}, "detail": {"pid": 8062, "restart_count": 15, "start_count": 16}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 15, "h264_decode_error_count": 15, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:41:47+09:00", "last_error_summary": "[h264 @ 0x562f20eabf80] error while decoding MB 62 30, bytestream -9", "last_exit_at": "2026-07-02T12:41:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T12:41:49+09:00", "pid": 8062, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.859, "name": "ffmpeg", "pid": 8062, "rss_mb": 57.859, "state": "S (sleeping)", "threads": 1}, "restart_count": 15, "restart_count_per_hour": 5, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 16}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.48, "sample_count": 2267, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 97206, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:41:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 4.33, "last_frame_pushed_at": "2026-07-02T12:41:47+09:00", "last_frame_read_age_sec": 4.455, "last_frame_read_at": "2026-07-02T12:41:47+09:00", "mjpeg": {"age_sec": 4.31, "exists": true, "mtime": "2026-07-02T12:41:47.509051+09:00", "mtime_epoch": 1782963707.5090513, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 234832}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T12:41:51+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1077.918, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 196, "pushed_frames": 196, "read_frames": 195, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.537, "pushed": 6.537, "read": 6.504}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.02, "count": 195, "max_ms": 8.153, "p95_ms": 6.832}, "gstreamer_push": {"avg_ms": 1.117, "count": 196, "max_ms": 1.535, "p95_ms": 1.231}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.385, "count": 195, "max_ms": 80.759, "p95_ms": 71.823}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.159, "count": 195, "max_ms": 109.361, "p95_ms": 90.737}, "yolo11_640_fire_11class": {"avg_ms": 56.397, "count": 195, "max_ms": 84.111, "p95_ms": 73.504}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.25, "count": 195, "max_ms": 100.419, "p95_ms": 91.664}}, "truncated_count": 0}, "processing_total": {"avg_ms": 118.538, "count": 196, "max_ms": 162.518, "p95_ms": 148.262}, "read_interval": {"avg_ms": 131.483, "count": 195, "max_ms": 177.378, "p95_ms": 162.913}, "triton_wait": {"avg_ms": 64.505, "count": 195, "max_ms": 109.374, "p95_ms": 93.32}, "visualize": {"avg_ms": 5.202, "count": 195, "max_ms": 10.978, "p95_ms": 5.806}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.982, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 29, "tracking_summary": 814}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5061.641, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55c464c7ff80] error while decoding MB 3 46, bytestream -25 [in#0/rtsp @ 0x55c464bade00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c464bade00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 16, "h264_decode_error_count": 16, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:43:26+09:00", "last_error_summary": "[h264 @ 0x55c464c7ff80] error while decoding MB 3 46, bytestream -25 [in#0/rtsp @ 0x55c464bade00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c464bade00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T12:41:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 96.386, "last_started_at": "2026-07-02T12:41:49+09:00", "pid": 8062, "process": {"exists": false, "pid": 8062}, "restart_count": 15, "restart_count_per_hour": 4, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 16}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 158.171, "sample_count": 2024, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 97685, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:43:25+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.076, "last_frame_pushed_at": "2026-07-02T12:43:26+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-02T12:43:26+09:00", "mjpeg": {"age_sec": 0.057, "exists": true, "mtime": "2026-07-02T12:43:26.147087+09:00", "mtime_epoch": 1782963806.1470873, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 241018}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T12:43:26+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1078.176, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 43, "pushed_frames": 44, "read_frames": 44, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.433, "pushed": 1.467, "read": 1.467}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.814, "count": 43, "max_ms": 8.477, "p95_ms": 7.799}, "gstreamer_push": {"avg_ms": 1.203, "count": 44, "max_ms": 1.74, "p95_ms": 1.395}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.436, "count": 43, "max_ms": 72.125, "p95_ms": 68.862}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.079, "count": 43, "max_ms": 88.405, "p95_ms": 77.149}, "yolo11_640_fire_11class": {"avg_ms": 56.165, "count": 43, "max_ms": 74.282, "p95_ms": 70.353}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.923, "count": 43, "max_ms": 87.884, "p95_ms": 82.218}}, "truncated_count": 0}, "processing_total": {"avg_ms": 106.764, "count": 43, "max_ms": 143.152, "p95_ms": 138.428}, "read_interval": {"avg_ms": 682.605, "count": 44, "max_ms": 15893.175, "p95_ms": 156.256}, "triton_wait": {"avg_ms": 62.578, "count": 43, "max_ms": 88.417, "p95_ms": 84.645}, "visualize": {"avg_ms": 4.846, "count": 43, "max_ms": 6.223, "p95_ms": 5.794}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 29, "tracking_summary": 814}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5060.57, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T12:43:26+09:00", "last_error_summary": "[h264 @ 0x55c464c7ff80] error while decoding MB 3 46, bytestream -25 [in#0/rtsp @ 0x55c464bade00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c464bade00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 16, "h264_decode_error_count": 16, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:43:26+09:00", "last_error_summary": "[h264 @ 0x55c464c7ff80] error while decoding MB 3 46, bytestream -25 [in#0/rtsp @ 0x55c464bade00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c464bade00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T12:43:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 96.442, "last_started_at": "2026-07-02T12:41:49+09:00", "pid": 8062, "process": {"exists": false, "pid": 8062}, "restart_count": 15, "restart_count_per_hour": 4, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 16}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 158.171, "sample_count": 2025, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 97686, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:43:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T12:43:26+09:00", "last_frame_read_age_sec": 0.121, "last_frame_read_at": "2026-07-02T12:43:26+09:00", "mjpeg": {"age_sec": 0.113, "exists": true, "mtime": "2026-07-02T12:43:26.147087+09:00", "mtime_epoch": 1782963806.1470873, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 241018}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T12:43:26+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1078.176, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 44, "pushed_frames": 44, "read_frames": 43, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.47, "pushed": 1.47, "read": 1.436}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.821, "count": 44, "max_ms": 8.477, "p95_ms": 7.799}, "gstreamer_push": {"avg_ms": 1.198, "count": 44, "max_ms": 1.74, "p95_ms": 1.395}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.415, "count": 44, "max_ms": 72.125, "p95_ms": 68.862}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.78, "count": 44, "max_ms": 88.405, "p95_ms": 77.149}, "yolo11_640_fire_11class": {"avg_ms": 56.073, "count": 44, "max_ms": 74.282, "p95_ms": 70.353}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.834, "count": 44, "max_ms": 87.884, "p95_ms": 82.218}}, "truncated_count": 0}, "processing_total": {"avg_ms": 106.939, "count": 44, "max_ms": 143.152, "p95_ms": 138.428}, "read_interval": {"avg_ms": 695.839, "count": 43, "max_ms": 15893.175, "p95_ms": 156.256}, "triton_wait": {"avg_ms": 62.429, "count": 44, "max_ms": 88.417, "p95_ms": 84.645}, "visualize": {"avg_ms": 4.891, "count": 44, "max_ms": 6.811, "p95_ms": 6.189}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.941, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 29, "tracking_summary": 814}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5060.672, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 16, "h264_decode_error_count": 16, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:43:26+09:00", "last_error_summary": "[h264 @ 0x55c464c7ff80] error while decoding MB 3 46, bytestream -25 [in#0/rtsp @ 0x55c464bade00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c464bade00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T12:43:26+09:00", "last_returncode": 0, "last_stable_duration_sec": 96.447, "last_started_at": "2026-07-02T12:41:49+09:00", "pid": 8062, "process": {"exists": false, "pid": 8062}, "restart_count": 15, "restart_count_per_hour": 4, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 16}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 158.171, "sample_count": 2024, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 97686, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:43:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-02T12:43:26+09:00", "last_frame_read_age_sec": 0.126, "last_frame_read_at": "2026-07-02T12:43:26+09:00", "mjpeg": {"age_sec": 0.118, "exists": true, "mtime": "2026-07-02T12:43:26.147087+09:00", "mtime_epoch": 1782963806.1470873, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 241018}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T12:43:26+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1078.176, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 44, "pushed_frames": 44, "read_frames": 43, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.469, "pushed": 1.469, "read": 1.436}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.821, "count": 44, "max_ms": 8.477, "p95_ms": 7.799}, "gstreamer_push": {"avg_ms": 1.198, "count": 44, "max_ms": 1.74, "p95_ms": 1.395}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.415, "count": 44, "max_ms": 72.125, "p95_ms": 68.862}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.78, "count": 44, "max_ms": 88.405, "p95_ms": 77.149}, "yolo11_640_fire_11class": {"avg_ms": 56.073, "count": 44, "max_ms": 74.282, "p95_ms": 70.353}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.834, "count": 44, "max_ms": 87.884, "p95_ms": 82.218}}, "truncated_count": 0}, "processing_total": {"avg_ms": 106.939, "count": 44, "max_ms": 143.152, "p95_ms": 138.428}, "read_interval": {"avg_ms": 695.839, "count": 43, "max_ms": 15893.175, "p95_ms": 156.256}, "triton_wait": {"avg_ms": 62.429, "count": 44, "max_ms": 88.417, "p95_ms": 84.645}, "visualize": {"avg_ms": 4.891, "count": 44, "max_ms": 6.811, "p95_ms": 6.189}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.946, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 29, "tracking_summary": 814}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5231.82, "limit_mb": 8796093022207.996}, "detail": {"pid": 8214, "restart_count": 16, "start_count": 17}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 16, "h264_decode_error_count": 16, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:43:26+09:00", "last_error_summary": "[h264 @ 0x55c464c7ff80] error while decoding MB 3 46, bytestream -25 [in#0/rtsp @ 0x55c464bade00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c464bade00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T12:43:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T12:43:28+09:00", "pid": 8214, "process": {"exists": true, "fd_count": 5, "hwm_mb": 206.781, "name": "ffmpeg", "pid": 8214, "rss_mb": 206.781, "state": "S (sleeping)", "threads": 106}, "restart_count": 16, "restart_count_per_hour": 5, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 17}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 158.259, "sample_count": 1989, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 97686, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:43:26+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 4.47, "last_frame_pushed_at": "2026-07-02T12:43:26+09:00", "last_frame_read_age_sec": 4.586, "last_frame_read_at": "2026-07-02T12:43:26+09:00", "mjpeg": {"age_sec": 4.45, "exists": true, "mtime": "2026-07-02T12:43:26.275090+09:00", "mtime_epoch": 1782963806.27509, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 239463}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T12:43:30+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1078.176, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.636, "pushed": 0.636, "read": 0.636}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.35, "count": 14, "max_ms": 7.799, "p95_ms": 7.799}, "gstreamer_push": {"avg_ms": 1.168, "count": 14, "max_ms": 1.353, "p95_ms": 1.353}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.59, "count": 14, "max_ms": 62.99, "p95_ms": 62.99}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.523, "count": 14, "max_ms": 78.143, "p95_ms": 78.143}, "yolo11_640_fire_11class": {"avg_ms": 55.536, "count": 14, "max_ms": 70.438, "p95_ms": 70.438}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.242, "count": 14, "max_ms": 77.489, "p95_ms": 77.489}}, "truncated_count": 0}, "processing_total": {"avg_ms": 117.759, "count": 14, "max_ms": 143.152, "p95_ms": 143.152}, "read_interval": {"avg_ms": 1898.053, "count": 14, "max_ms": 15893.175, "p95_ms": 15893.175}, "triton_wait": {"avg_ms": 62.513, "count": 14, "max_ms": 78.153, "p95_ms": 78.153}, "visualize": {"avg_ms": 5.308, "count": 14, "max_ms": 6.811, "p95_ms": 6.811}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.017, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 29, "tracking_summary": 816}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5068.613, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55703c78e640] error while decoding MB 46 54, bytestream -41 [in#0/rtsp @ 0x55703c651e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55703c651e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 17, "h264_decode_error_count": 17, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:44:48+09:00", "last_error_summary": "[h264 @ 0x55703c78e640] error while decoding MB 46 54, bytestream -41 [in#0/rtsp @ 0x55703c651e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55703c651e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T12:43:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 79.975, "last_started_at": "2026-07-02T12:43:28+09:00", "pid": 8214, "process": {"exists": false, "pid": 8214}, "restart_count": 16, "restart_count_per_hour": 5, "rtsp_timeout_count": 20, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 17}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 162.092, "sample_count": 1740, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 98035, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:44:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.072, "last_frame_pushed_at": "2026-07-02T12:44:48+09:00", "last_frame_read_age_sec": 0.062, "last_frame_read_at": "2026-07-02T12:44:48+09:00", "mjpeg": {"age_sec": 0.054, "exists": true, "mtime": "2026-07-02T12:44:48.644789+09:00", "mtime_epoch": 1782963888.6447892, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 238821}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T12:44:48+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1078.434, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.737, "pushed": 0.737, "read": 0.794}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.006, "count": 13, "max_ms": 7.47, "p95_ms": 7.47}, "gstreamer_push": {"avg_ms": 1.153, "count": 13, "max_ms": 1.39, "p95_ms": 1.39}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.465, "count": 13, "max_ms": 74.812, "p95_ms": 74.812}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.171, "count": 13, "max_ms": 96.574, "p95_ms": 96.574}, "yolo11_640_fire_11class": {"avg_ms": 59.881, "count": 13, "max_ms": 76.668, "p95_ms": 76.668}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.689, "count": 13, "max_ms": 91.647, "p95_ms": 91.647}}, "truncated_count": 0}, "processing_total": {"avg_ms": 114.12, "count": 13, "max_ms": 160.682, "p95_ms": 160.682}, "read_interval": {"avg_ms": 2332.372, "count": 14, "max_ms": 16054.951, "p95_ms": 16054.951}, "triton_wait": {"avg_ms": 68.768, "count": 13, "max_ms": 96.586, "p95_ms": 96.586}, "visualize": {"avg_ms": 4.917, "count": 13, "max_ms": 6.196, "p95_ms": 6.196}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.641, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 29, "tracking_summary": 816}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5068.023, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T12:44:48+09:00", "last_error_summary": "[h264 @ 0x55703c78e640] error while decoding MB 46 54, bytestream -41 [in#0/rtsp @ 0x55703c651e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55703c651e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 17, "h264_decode_error_count": 17, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:44:48+09:00", "last_error_summary": "[h264 @ 0x55703c78e640] error while decoding MB 46 54, bytestream -41 [in#0/rtsp @ 0x55703c651e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55703c651e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T12:44:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 79.99, "last_started_at": "2026-07-02T12:43:28+09:00", "pid": 8214, "process": {"exists": false, "pid": 8214}, "restart_count": 16, "restart_count_per_hour": 5, "rtsp_timeout_count": 20, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 17}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 162.092, "sample_count": 1740, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 98036, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:44:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T12:44:48+09:00", "last_frame_read_age_sec": 0.077, "last_frame_read_at": "2026-07-02T12:44:48+09:00", "mjpeg": {"age_sec": 0.07, "exists": true, "mtime": "2026-07-02T12:44:48.644789+09:00", "mtime_epoch": 1782963888.6447892, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 238821}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T12:44:48+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1078.434, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.793, "pushed": 0.793, "read": 0.793}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.849, "count": 14, "max_ms": 7.47, "p95_ms": 7.47}, "gstreamer_push": {"avg_ms": 1.16, "count": 14, "max_ms": 1.39, "p95_ms": 1.39}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.854, "count": 14, "max_ms": 74.812, "p95_ms": 74.812}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.549, "count": 14, "max_ms": 96.574, "p95_ms": 96.574}, "yolo11_640_fire_11class": {"avg_ms": 58.9, "count": 14, "max_ms": 76.668, "p95_ms": 76.668}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.947, "count": 14, "max_ms": 91.647, "p95_ms": 91.647}}, "truncated_count": 0}, "processing_total": {"avg_ms": 110.981, "count": 14, "max_ms": 160.682, "p95_ms": 160.682}, "read_interval": {"avg_ms": 2332.372, "count": 14, "max_ms": 16054.951, "p95_ms": 16054.951}, "triton_wait": {"avg_ms": 67.807, "count": 14, "max_ms": 96.586, "p95_ms": 96.586}, "visualize": {"avg_ms": 4.857, "count": 14, "max_ms": 6.196, "p95_ms": 6.196}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.656, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 29, "tracking_summary": 816}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5068.008, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 17, "h264_decode_error_count": 17, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:44:48+09:00", "last_error_summary": "[h264 @ 0x55703c78e640] error while decoding MB 46 54, bytestream -41 [in#0/rtsp @ 0x55703c651e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55703c651e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T12:44:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 79.997, "last_started_at": "2026-07-02T12:43:28+09:00", "pid": 8214, "process": {"exists": false, "pid": 8214}, "restart_count": 16, "restart_count_per_hour": 5, "rtsp_timeout_count": 20, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 17}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 162.092, "sample_count": 1740, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 98036, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:44:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-02T12:44:48+09:00", "last_frame_read_age_sec": 0.084, "last_frame_read_at": "2026-07-02T12:44:48+09:00", "mjpeg": {"age_sec": 0.076, "exists": true, "mtime": "2026-07-02T12:44:48.644789+09:00", "mtime_epoch": 1782963888.6447892, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 238821}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T12:44:48+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1078.434, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.793, "pushed": 0.793, "read": 0.793}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.849, "count": 14, "max_ms": 7.47, "p95_ms": 7.47}, "gstreamer_push": {"avg_ms": 1.16, "count": 14, "max_ms": 1.39, "p95_ms": 1.39}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.854, "count": 14, "max_ms": 74.812, "p95_ms": 74.812}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.549, "count": 14, "max_ms": 96.574, "p95_ms": 96.574}, "yolo11_640_fire_11class": {"avg_ms": 58.9, "count": 14, "max_ms": 76.668, "p95_ms": 76.668}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.947, "count": 14, "max_ms": 91.647, "p95_ms": 91.647}}, "truncated_count": 0}, "processing_total": {"avg_ms": 110.981, "count": 14, "max_ms": 160.682, "p95_ms": 160.682}, "read_interval": {"avg_ms": 2332.372, "count": 14, "max_ms": 16054.951, "p95_ms": 16054.951}, "triton_wait": {"avg_ms": 67.807, "count": 14, "max_ms": 96.586, "p95_ms": 96.586}, "visualize": {"avg_ms": 4.857, "count": 14, "max_ms": 6.196, "p95_ms": 6.196}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.662, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 29, "tracking_summary": 816}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5236.512, "limit_mb": 8796093022207.996}, "detail": {"pid": 8366, "restart_count": 17, "start_count": 18}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 17, "h264_decode_error_count": 17, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:44:48+09:00", "last_error_summary": "[h264 @ 0x55703c78e640] error while decoding MB 46 54, bytestream -41 [in#0/rtsp @ 0x55703c651e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55703c651e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T12:44:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T12:44:51+09:00", "pid": 8366, "process": {"exists": true, "fd_count": 5, "hwm_mb": 207.031, "name": "ffmpeg", "pid": 8366, "rss_mb": 207.031, "state": "S (sleeping)", "threads": 106}, "restart_count": 17, "restart_count_per_hour": 6, "rtsp_timeout_count": 20, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 18}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 162.518, "sample_count": 1703, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 98036, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:44:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 4.894, "last_frame_pushed_at": "2026-07-02T12:44:48+09:00", "last_frame_read_age_sec": 4.966, "last_frame_read_at": "2026-07-02T12:44:48+09:00", "mjpeg": {"age_sec": 4.879, "exists": true, "mtime": "2026-07-02T12:44:48.724791+09:00", "mtime_epoch": 1782963888.7247908, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 240267}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T12:44:53+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1078.434, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 22, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.621, "pushed": 0.621, "read": 0.621}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.849, "count": 14, "max_ms": 7.47, "p95_ms": 7.47}, "gstreamer_push": {"avg_ms": 1.16, "count": 14, "max_ms": 1.39, "p95_ms": 1.39}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.854, "count": 14, "max_ms": 74.812, "p95_ms": 74.812}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.549, "count": 14, "max_ms": 96.574, "p95_ms": 96.574}, "yolo11_640_fire_11class": {"avg_ms": 58.9, "count": 14, "max_ms": 76.668, "p95_ms": 76.668}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.947, "count": 14, "max_ms": 91.647, "p95_ms": 91.647}}, "truncated_count": 0}, "processing_total": {"avg_ms": 110.981, "count": 14, "max_ms": 160.682, "p95_ms": 160.682}, "read_interval": {"avg_ms": 2332.372, "count": 14, "max_ms": 16054.951, "p95_ms": 16054.951}, "triton_wait": {"avg_ms": 67.807, "count": 14, "max_ms": 96.586, "p95_ms": 96.586}, "visualize": {"avg_ms": 4.857, "count": 14, "max_ms": 6.196, "p95_ms": 6.196}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.545, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 32, "tracking_summary": 837}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5097.312, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5584cb457540] error while decoding MB 27 53, bytestream -23 [in#0/rtsp @ 0x5584cac14e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5584cac14e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 18, "h264_decode_error_count": 18, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:50:41+09:00", "last_error_summary": "[h264 @ 0x5584cb457540] error while decoding MB 27 53, bytestream -23 [in#0/rtsp @ 0x5584cac14e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5584cac14e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T12:44:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 349.813, "last_started_at": "2026-07-02T12:44:51+09:00", "pid": 8366, "process": {"exists": false, "pid": 8366}, "restart_count": 17, "restart_count_per_hour": 5, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 18}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 186.835, "sample_count": 2123, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 100475, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:50:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.078, "last_frame_pushed_at": "2026-07-02T12:50:41+09:00", "last_frame_read_age_sec": 0.062, "last_frame_read_at": "2026-07-02T12:50:41+09:00", "mjpeg": {"age_sec": 0.06, "exists": true, "mtime": "2026-07-02T12:50:41.356058+09:00", "mtime_epoch": 1782964241.3560581, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 243535}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T12:50:41+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1078.684, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 94, "pushed_frames": 94, "read_frames": 94, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.137, "pushed": 3.137, "read": 3.137}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.34, "count": 93, "max_ms": 8.695, "p95_ms": 6.692}, "gstreamer_push": {"avg_ms": 1.219, "count": 94, "max_ms": 1.741, "p95_ms": 1.422}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.013, "count": 93, "max_ms": 81.537, "p95_ms": 70.918}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.153, "count": 93, "max_ms": 96.62, "p95_ms": 90.988}, "yolo11_640_fire_11class": {"avg_ms": 57.45, "count": 93, "max_ms": 90.848, "p95_ms": 76.084}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.99, "count": 93, "max_ms": 95.796, "p95_ms": 81.919}}, "truncated_count": 0}, "processing_total": {"avg_ms": 100.794, "count": 94, "max_ms": 147.867, "p95_ms": 143.673}, "read_interval": {"avg_ms": 319.498, "count": 94, "max_ms": 10169.782, "p95_ms": 160.668}, "triton_wait": {"avg_ms": 65.672, "count": 93, "max_ms": 96.633, "p95_ms": 91.504}, "visualize": {"avg_ms": 4.583, "count": 93, "max_ms": 7.658, "p95_ms": 6.072}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.967, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 32, "tracking_summary": 837}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5096.969, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T12:50:41+09:00", "last_error_summary": "[h264 @ 0x5584cb457540] error while decoding MB 27 53, bytestream -23 [in#0/rtsp @ 0x5584cac14e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5584cac14e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 18, "h264_decode_error_count": 18, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:50:41+09:00", "last_error_summary": "[h264 @ 0x5584cb457540] error while decoding MB 27 53, bytestream -23 [in#0/rtsp @ 0x5584cac14e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5584cac14e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T12:50:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 349.851, "last_started_at": "2026-07-02T12:44:51+09:00", "pid": 8366, "process": {"exists": false, "pid": 8366}, "restart_count": 17, "restart_count_per_hour": 5, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 18}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 186.835, "sample_count": 2124, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 100476, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:50:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T12:50:41+09:00", "last_frame_read_age_sec": 0.1, "last_frame_read_at": "2026-07-02T12:50:41+09:00", "mjpeg": {"age_sec": 0.098, "exists": true, "mtime": "2026-07-02T12:50:41.356058+09:00", "mtime_epoch": 1782964241.3560581, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 243535}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T12:50:41+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1078.684, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 94, "pushed_frames": 94, "read_frames": 94, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.134, "pushed": 3.134, "read": 3.134}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.323, "count": 94, "max_ms": 8.695, "p95_ms": 6.692}, "gstreamer_push": {"avg_ms": 1.221, "count": 94, "max_ms": 1.741, "p95_ms": 1.422}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.16, "count": 94, "max_ms": 81.537, "p95_ms": 70.918}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.326, "count": 94, "max_ms": 96.62, "p95_ms": 90.988}, "yolo11_640_fire_11class": {"avg_ms": 57.627, "count": 94, "max_ms": 90.848, "p95_ms": 76.084}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.186, "count": 94, "max_ms": 95.796, "p95_ms": 81.919}}, "truncated_count": 0}, "processing_total": {"avg_ms": 100.405, "count": 94, "max_ms": 147.867, "p95_ms": 143.673}, "read_interval": {"avg_ms": 319.498, "count": 94, "max_ms": 10169.782, "p95_ms": 160.668}, "triton_wait": {"avg_ms": 65.818, "count": 94, "max_ms": 96.633, "p95_ms": 91.504}, "visualize": {"avg_ms": 4.574, "count": 94, "max_ms": 7.658, "p95_ms": 6.072}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 32, "tracking_summary": 837}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5097.012, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 18, "h264_decode_error_count": 18, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:50:41+09:00", "last_error_summary": "[h264 @ 0x5584cb457540] error while decoding MB 27 53, bytestream -23 [in#0/rtsp @ 0x5584cac14e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5584cac14e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T12:50:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 349.858, "last_started_at": "2026-07-02T12:44:51+09:00", "pid": 8366, "process": {"exists": false, "pid": 8366}, "restart_count": 17, "restart_count_per_hour": 5, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 18}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 186.835, "sample_count": 2123, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 100476, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:50:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-02T12:50:41+09:00", "last_frame_read_age_sec": 0.107, "last_frame_read_at": "2026-07-02T12:50:41+09:00", "mjpeg": {"age_sec": 0.105, "exists": true, "mtime": "2026-07-02T12:50:41.356058+09:00", "mtime_epoch": 1782964241.3560581, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 243535}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T12:50:41+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1078.684, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 94, "pushed_frames": 94, "read_frames": 94, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.134, "pushed": 3.134, "read": 3.134}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.323, "count": 94, "max_ms": 8.695, "p95_ms": 6.692}, "gstreamer_push": {"avg_ms": 1.221, "count": 94, "max_ms": 1.741, "p95_ms": 1.422}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.16, "count": 94, "max_ms": 81.537, "p95_ms": 70.918}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.326, "count": 94, "max_ms": 96.62, "p95_ms": 90.988}, "yolo11_640_fire_11class": {"avg_ms": 57.627, "count": 94, "max_ms": 90.848, "p95_ms": 76.084}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.186, "count": 94, "max_ms": 95.796, "p95_ms": 81.919}}, "truncated_count": 0}, "processing_total": {"avg_ms": 100.405, "count": 94, "max_ms": 147.867, "p95_ms": 143.673}, "read_interval": {"avg_ms": 319.498, "count": 94, "max_ms": 10169.782, "p95_ms": 160.668}, "triton_wait": {"avg_ms": 65.818, "count": 94, "max_ms": 96.633, "p95_ms": 91.504}, "visualize": {"avg_ms": 4.574, "count": 94, "max_ms": 7.658, "p95_ms": 6.072}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 32, "tracking_summary": 837}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5262.488, "limit_mb": 8796093022207.996}, "detail": {"pid": 8698, "restart_count": 18, "start_count": 19}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 18, "h264_decode_error_count": 18, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:50:41+09:00", "last_error_summary": "[h264 @ 0x5584cb457540] error while decoding MB 27 53, bytestream -23 [in#0/rtsp @ 0x5584cac14e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5584cac14e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T12:50:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T12:50:43+09:00", "pid": 8698, "process": {"exists": true, "fd_count": 5, "hwm_mb": 204.156, "name": "ffmpeg", "pid": 8698, "rss_mb": 204.156, "state": "S (sleeping)", "threads": 106}, "restart_count": 18, "restart_count_per_hour": 6, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 19}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 187.254, "sample_count": 2090, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 100476, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:50:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 4.112, "last_frame_pushed_at": "2026-07-02T12:50:41+09:00", "last_frame_read_age_sec": 4.206, "last_frame_read_at": "2026-07-02T12:50:41+09:00", "mjpeg": {"age_sec": 4.096, "exists": true, "mtime": "2026-07-02T12:50:41.464060+09:00", "mtime_epoch": 1782964241.4640603, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 244958}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T12:50:45+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1078.684, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 63, "pushed_frames": 63, "read_frames": 62, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.101, "pushed": 2.101, "read": 2.067}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.826, "count": 62, "max_ms": 8.695, "p95_ms": 6.335}, "gstreamer_push": {"avg_ms": 1.273, "count": 63, "max_ms": 1.741, "p95_ms": 1.424}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.61, "count": 62, "max_ms": 81.537, "p95_ms": 71.773}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.583, "count": 62, "max_ms": 96.62, "p95_ms": 90.537}, "yolo11_640_fire_11class": {"avg_ms": 58.354, "count": 62, "max_ms": 90.848, "p95_ms": 76.084}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.052, "count": 62, "max_ms": 95.796, "p95_ms": 81.086}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.483, "count": 63, "max_ms": 147.668, "p95_ms": 143.673}, "read_interval": {"avg_ms": 417.514, "count": 62, "max_ms": 10169.782, "p95_ms": 162.271}, "triton_wait": {"avg_ms": 66.417, "count": 62, "max_ms": 96.633, "p95_ms": 91.202}, "visualize": {"avg_ms": 4.221, "count": 62, "max_ms": 6.466, "p95_ms": 5.218}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.99, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 32, "tracking_summary": 850}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5097.387, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55c679174240] error while decoding MB 92 33, bytestream -5 [in#0/rtsp @ 0x55c678fa9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c678fa9e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 19, "h264_decode_error_count": 19, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:54:05+09:00", "last_error_summary": "[h264 @ 0x55c679174240] error while decoding MB 92 33, bytestream -5 [in#0/rtsp @ 0x55c678fa9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c678fa9e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T12:50:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 202.367, "last_started_at": "2026-07-02T12:50:43+09:00", "pid": 8698, "process": {"exists": false, "pid": 8698}, "restart_count": 18, "restart_count_per_hour": 6, "rtsp_timeout_count": 24, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 19}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 179.622, "sample_count": 2094, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 102055, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:54:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.075, "last_frame_pushed_at": "2026-07-02T12:54:05+09:00", "last_frame_read_age_sec": 0.062, "last_frame_read_at": "2026-07-02T12:54:05+09:00", "mjpeg": {"age_sec": 0.058, "exists": true, "mtime": "2026-07-02T12:54:05.868269+09:00", "mtime_epoch": 1782964445.8682692, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 233136}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T12:54:05+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1079.457, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.68, "pushed": 0.68, "read": 0.737}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.264, "count": 12, "max_ms": 6.244, "p95_ms": 6.244}, "gstreamer_push": {"avg_ms": 1.905, "count": 12, "max_ms": 2.361, "p95_ms": 2.361}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.623, "count": 12, "max_ms": 67.828, "p95_ms": 67.828}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.513, "count": 12, "max_ms": 88.419, "p95_ms": 88.419}, "yolo11_640_fire_11class": {"avg_ms": 64.115, "count": 12, "max_ms": 80.258, "p95_ms": 80.258}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 72.439, "count": 12, "max_ms": 84.094, "p95_ms": 84.094}}, "truncated_count": 0}, "processing_total": {"avg_ms": 92.377, "count": 12, "max_ms": 109.974, "p95_ms": 109.974}, "read_interval": {"avg_ms": 2355.469, "count": 13, "max_ms": 16371.725, "p95_ms": 16371.725}, "triton_wait": {"avg_ms": 75.019, "count": 12, "max_ms": 88.433, "p95_ms": 88.433}, "visualize": {"avg_ms": 5.306, "count": 12, "max_ms": 6.901, "p95_ms": 6.901}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.646, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 32, "tracking_summary": 850}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5096.891, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T12:54:05+09:00", "last_error_summary": "[h264 @ 0x55c679174240] error while decoding MB 92 33, bytestream -5 [in#0/rtsp @ 0x55c678fa9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c678fa9e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 19, "h264_decode_error_count": 19, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:54:05+09:00", "last_error_summary": "[h264 @ 0x55c679174240] error while decoding MB 92 33, bytestream -5 [in#0/rtsp @ 0x55c678fa9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c678fa9e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T12:54:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 202.383, "last_started_at": "2026-07-02T12:50:43+09:00", "pid": 8698, "process": {"exists": false, "pid": 8698}, "restart_count": 18, "restart_count_per_hour": 6, "rtsp_timeout_count": 24, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 19}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 179.622, "sample_count": 2095, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 102056, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:54:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T12:54:05+09:00", "last_frame_read_age_sec": 0.078, "last_frame_read_at": "2026-07-02T12:54:05+09:00", "mjpeg": {"age_sec": 0.073, "exists": true, "mtime": "2026-07-02T12:54:05.868269+09:00", "mtime_epoch": 1782964445.8682692, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 233136}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T12:54:05+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1079.457, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.736, "pushed": 0.736, "read": 0.736}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.165, "count": 13, "max_ms": 6.244, "p95_ms": 6.244}, "gstreamer_push": {"avg_ms": 1.862, "count": 13, "max_ms": 2.361, "p95_ms": 2.361}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.3, "count": 13, "max_ms": 67.828, "p95_ms": 67.828}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.997, "count": 13, "max_ms": 88.419, "p95_ms": 88.419}, "yolo11_640_fire_11class": {"avg_ms": 63.48, "count": 13, "max_ms": 80.258, "p95_ms": 80.258}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.013, "count": 13, "max_ms": 84.094, "p95_ms": 84.094}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.689, "count": 13, "max_ms": 109.974, "p95_ms": 109.974}, "read_interval": {"avg_ms": 2355.469, "count": 13, "max_ms": 16371.725, "p95_ms": 16371.725}, "triton_wait": {"avg_ms": 73.546, "count": 13, "max_ms": 88.433, "p95_ms": 88.433}, "visualize": {"avg_ms": 5.197, "count": 13, "max_ms": 6.901, "p95_ms": 6.901}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.661, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 32, "tracking_summary": 850}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5096.707, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 19, "h264_decode_error_count": 19, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:54:05+09:00", "last_error_summary": "[h264 @ 0x55c679174240] error while decoding MB 92 33, bytestream -5 [in#0/rtsp @ 0x55c678fa9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c678fa9e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T12:54:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 202.39, "last_started_at": "2026-07-02T12:50:43+09:00", "pid": 8698, "process": {"exists": false, "pid": 8698}, "restart_count": 18, "restart_count_per_hour": 6, "rtsp_timeout_count": 24, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 19}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 179.622, "sample_count": 2095, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 102056, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:54:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-02T12:54:05+09:00", "last_frame_read_age_sec": 0.085, "last_frame_read_at": "2026-07-02T12:54:05+09:00", "mjpeg": {"age_sec": 0.081, "exists": true, "mtime": "2026-07-02T12:54:05.868269+09:00", "mtime_epoch": 1782964445.8682692, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 233136}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T12:54:05+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1079.457, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.736, "pushed": 0.736, "read": 0.736}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.165, "count": 13, "max_ms": 6.244, "p95_ms": 6.244}, "gstreamer_push": {"avg_ms": 1.862, "count": 13, "max_ms": 2.361, "p95_ms": 2.361}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.3, "count": 13, "max_ms": 67.828, "p95_ms": 67.828}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.997, "count": 13, "max_ms": 88.419, "p95_ms": 88.419}, "yolo11_640_fire_11class": {"avg_ms": 63.48, "count": 13, "max_ms": 80.258, "p95_ms": 80.258}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.013, "count": 13, "max_ms": 84.094, "p95_ms": 84.094}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.689, "count": 13, "max_ms": 109.974, "p95_ms": 109.974}, "read_interval": {"avg_ms": 2355.469, "count": 13, "max_ms": 16371.725, "p95_ms": 16371.725}, "triton_wait": {"avg_ms": 73.546, "count": 13, "max_ms": 88.433, "p95_ms": 88.433}, "visualize": {"avg_ms": 5.197, "count": 13, "max_ms": 6.901, "p95_ms": 6.901}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.669, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 32, "tracking_summary": 850}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5121.082, "limit_mb": 8796093022207.996}, "detail": {"pid": 8850, "restart_count": 19, "start_count": 20}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 19, "h264_decode_error_count": 19, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:54:05+09:00", "last_error_summary": "[h264 @ 0x55c679174240] error while decoding MB 92 33, bytestream -5 [in#0/rtsp @ 0x55c678fa9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c678fa9e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T12:54:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T12:54:07+09:00", "pid": 8850, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.461, "name": "ffmpeg", "pid": 8850, "rss_mb": 58.461, "state": "S (sleeping)", "threads": 1}, "restart_count": 19, "restart_count_per_hour": 7, "rtsp_timeout_count": 24, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 20}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 174.717, "sample_count": 2074, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 102056, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:54:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 4.044, "last_frame_pushed_at": "2026-07-02T12:54:05+09:00", "last_frame_read_age_sec": 4.117, "last_frame_read_at": "2026-07-02T12:54:05+09:00", "mjpeg": {"age_sec": 4.028, "exists": true, "mtime": "2026-07-02T12:54:05.952271+09:00", "mtime_epoch": 1782964445.952271, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 233018}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T12:54:09+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1079.457, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.599, "pushed": 0.599, "read": 0.599}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.165, "count": 13, "max_ms": 6.244, "p95_ms": 6.244}, "gstreamer_push": {"avg_ms": 1.862, "count": 13, "max_ms": 2.361, "p95_ms": 2.361}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.3, "count": 13, "max_ms": 67.828, "p95_ms": 67.828}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.997, "count": 13, "max_ms": 88.419, "p95_ms": 88.419}, "yolo11_640_fire_11class": {"avg_ms": 63.48, "count": 13, "max_ms": 80.258, "p95_ms": 80.258}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 71.013, "count": 13, "max_ms": 84.094, "p95_ms": 84.094}}, "truncated_count": 0}, "processing_total": {"avg_ms": 90.689, "count": 13, "max_ms": 109.974, "p95_ms": 109.974}, "read_interval": {"avg_ms": 2355.469, "count": 13, "max_ms": 16371.725, "p95_ms": 16371.725}, "triton_wait": {"avg_ms": 73.546, "count": 13, "max_ms": 88.433, "p95_ms": 88.433}, "visualize": {"avg_ms": 5.197, "count": 13, "max_ms": 6.901, "p95_ms": 6.901}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.7, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 32, "tracking_summary": 869}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5109.957, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55c41feeb700] error while decoding MB 22 34, bytestream -35 [in#0/rtsp @ 0x55c41fe66e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c41fe66e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 20, "h264_decode_error_count": 20, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:59:19+09:00", "last_error_summary": "[h264 @ 0x55c41feeb700] error while decoding MB 22 34, bytestream -35 [in#0/rtsp @ 0x55c41fe66e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c41fe66e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T12:54:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 311.101, "last_started_at": "2026-07-02T12:54:07+09:00", "pid": 8850, "process": {"exists": false, "pid": 8850}, "restart_count": 19, "restart_count_per_hour": 6, "rtsp_timeout_count": 26, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 20}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 146.141, "sample_count": 2171, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 104297, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:59:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.074, "last_frame_pushed_at": "2026-07-02T12:59:19+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-02T12:59:19+09:00", "mjpeg": {"age_sec": 0.054, "exists": true, "mtime": "2026-07-02T12:59:19.026715+09:00", "mtime_epoch": 1782964759.0267146, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 249775}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T12:59:19+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1080.488, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 88, "pushed_frames": 88, "read_frames": 88, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.936, "pushed": 2.936, "read": 2.936}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.205, "count": 88, "max_ms": 9.397, "p95_ms": 7.592}, "gstreamer_push": {"avg_ms": 1.156, "count": 88, "max_ms": 1.453, "p95_ms": 1.377}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.919, "count": 88, "max_ms": 78.652, "p95_ms": 68.582}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.738, "count": 88, "max_ms": 111.433, "p95_ms": 91.056}, "yolo11_640_fire_11class": {"avg_ms": 56.489, "count": 88, "max_ms": 82.02, "p95_ms": 73.098}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.13, "count": 88, "max_ms": 97.895, "p95_ms": 84.995}}, "truncated_count": 0}, "processing_total": {"avg_ms": 110.508, "count": 88, "max_ms": 162.962, "p95_ms": 141.69}, "read_interval": {"avg_ms": 341.3, "count": 88, "max_ms": 10162.202, "p95_ms": 157.874}, "triton_wait": {"avg_ms": 64.489, "count": 88, "max_ms": 111.447, "p95_ms": 91.068}, "visualize": {"avg_ms": 4.878, "count": 88, "max_ms": 6.544, "p95_ms": 5.793}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.969, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 32, "tracking_summary": 869}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5109.238, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T12:59:19+09:00", "last_error_summary": "[h264 @ 0x55c41feeb700] error while decoding MB 22 34, bytestream -35 [in#0/rtsp @ 0x55c41fe66e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c41fe66e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 20, "h264_decode_error_count": 20, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:59:19+09:00", "last_error_summary": "[h264 @ 0x55c41feeb700] error while decoding MB 22 34, bytestream -35 [in#0/rtsp @ 0x55c41fe66e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c41fe66e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T12:59:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 311.165, "last_started_at": "2026-07-02T12:54:07+09:00", "pid": 8850, "process": {"exists": false, "pid": 8850}, "restart_count": 19, "restart_count_per_hour": 6, "rtsp_timeout_count": 26, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 20}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 146.098, "sample_count": 2172, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 104297, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:59:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T12:59:19+09:00", "last_frame_read_age_sec": 0.128, "last_frame_read_at": "2026-07-02T12:59:19+09:00", "mjpeg": {"age_sec": 0.118, "exists": true, "mtime": "2026-07-02T12:59:19.026715+09:00", "mtime_epoch": 1782964759.0267146, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 249775}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T12:59:19+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1080.492, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 88, "pushed_frames": 88, "read_frames": 87, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.944, "pushed": 2.944, "read": 2.91}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.205, "count": 88, "max_ms": 9.397, "p95_ms": 7.592}, "gstreamer_push": {"avg_ms": 1.156, "count": 88, "max_ms": 1.453, "p95_ms": 1.377}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.919, "count": 88, "max_ms": 78.652, "p95_ms": 68.582}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.738, "count": 88, "max_ms": 111.433, "p95_ms": 91.056}, "yolo11_640_fire_11class": {"avg_ms": 56.489, "count": 88, "max_ms": 82.02, "p95_ms": 73.098}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.13, "count": 88, "max_ms": 97.895, "p95_ms": 84.995}}, "truncated_count": 0}, "processing_total": {"avg_ms": 110.386, "count": 88, "max_ms": 162.962, "p95_ms": 141.69}, "read_interval": {"avg_ms": 343.559, "count": 87, "max_ms": 10162.202, "p95_ms": 157.874}, "triton_wait": {"avg_ms": 64.489, "count": 88, "max_ms": 111.447, "p95_ms": 91.068}, "visualize": {"avg_ms": 4.878, "count": 88, "max_ms": 6.544, "p95_ms": 5.793}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.893, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 32, "tracking_summary": 869}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5109.355, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 20, "h264_decode_error_count": 20, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:59:19+09:00", "last_error_summary": "[h264 @ 0x55c41feeb700] error while decoding MB 22 34, bytestream -35 [in#0/rtsp @ 0x55c41fe66e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c41fe66e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T12:59:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 311.171, "last_started_at": "2026-07-02T12:54:07+09:00", "pid": 8850, "process": {"exists": false, "pid": 8850}, "restart_count": 19, "restart_count_per_hour": 6, "rtsp_timeout_count": 26, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 20}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 146.098, "sample_count": 2172, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 104297, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:59:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-02T12:59:19+09:00", "last_frame_read_age_sec": 0.134, "last_frame_read_at": "2026-07-02T12:59:19+09:00", "mjpeg": {"age_sec": 0.123, "exists": true, "mtime": "2026-07-02T12:59:19.026715+09:00", "mtime_epoch": 1782964759.0267146, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 249775}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T12:59:19+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1080.492, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 88, "pushed_frames": 88, "read_frames": 87, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.943, "pushed": 2.943, "read": 2.91}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.205, "count": 88, "max_ms": 9.397, "p95_ms": 7.592}, "gstreamer_push": {"avg_ms": 1.156, "count": 88, "max_ms": 1.453, "p95_ms": 1.377}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.919, "count": 88, "max_ms": 78.652, "p95_ms": 68.582}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.738, "count": 88, "max_ms": 111.433, "p95_ms": 91.056}, "yolo11_640_fire_11class": {"avg_ms": 56.489, "count": 88, "max_ms": 82.02, "p95_ms": 73.098}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.13, "count": 88, "max_ms": 97.895, "p95_ms": 84.995}}, "truncated_count": 0}, "processing_total": {"avg_ms": 110.386, "count": 88, "max_ms": 162.962, "p95_ms": 141.69}, "read_interval": {"avg_ms": 343.559, "count": 87, "max_ms": 10162.202, "p95_ms": 157.874}, "triton_wait": {"avg_ms": 64.489, "count": 88, "max_ms": 111.447, "p95_ms": 91.068}, "visualize": {"avg_ms": 4.878, "count": 88, "max_ms": 6.544, "p95_ms": 5.793}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.899, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 32, "tracking_summary": 869}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5128.359, "limit_mb": 8796093022207.996}, "detail": {"pid": 9002, "restart_count": 20, "start_count": 21}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 20, "h264_decode_error_count": 20, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T12:59:19+09:00", "last_error_summary": "[h264 @ 0x55c41feeb700] error while decoding MB 22 34, bytestream -35 [in#0/rtsp @ 0x55c41fe66e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55c41fe66e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T12:59:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T12:59:22+09:00", "pid": 9002, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.484, "name": "ffmpeg", "pid": 9002, "rss_mb": 58.484, "state": "S (sleeping)", "threads": 1}, "restart_count": 20, "restart_count_per_hour": 7, "rtsp_timeout_count": 26, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 21}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 146.231, "sample_count": 2125, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 104297, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T12:59:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 4.875, "last_frame_pushed_at": "2026-07-02T12:59:19+09:00", "last_frame_read_age_sec": 4.996, "last_frame_read_at": "2026-07-02T12:59:19+09:00", "mjpeg": {"age_sec": 4.857, "exists": true, "mtime": "2026-07-02T12:59:19.154717+09:00", "mtime_epoch": 1782964759.1547172, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 245659}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T12:59:24+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1080.492, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 53, "pushed_frames": 53, "read_frames": 52, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.768, "pushed": 1.768, "read": 1.734}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.936, "count": 52, "max_ms": 8.746, "p95_ms": 8.009}, "gstreamer_push": {"avg_ms": 1.181, "count": 53, "max_ms": 1.453, "p95_ms": 1.389}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.239, "count": 52, "max_ms": 68.582, "p95_ms": 66.283}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.253, "count": 52, "max_ms": 97.164, "p95_ms": 82.221}, "yolo11_640_fire_11class": {"avg_ms": 54.019, "count": 52, "max_ms": 73.098, "p95_ms": 70.054}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.752, "count": 52, "max_ms": 86.733, "p95_ms": 81.747}}, "truncated_count": 0}, "processing_total": {"avg_ms": 101.335, "count": 53, "max_ms": 145.029, "p95_ms": 137.831}, "read_interval": {"avg_ms": 482.659, "count": 52, "max_ms": 10162.202, "p95_ms": 155.386}, "triton_wait": {"avg_ms": 59.811, "count": 52, "max_ms": 97.176, "p95_ms": 86.745}, "visualize": {"avg_ms": 4.694, "count": 52, "max_ms": 6.544, "p95_ms": 5.979}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.983, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 869}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5119.301, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55bb528a98c0] left block unavailable for requested intra mode [h264 @ 0x55bb528a98c0] error while decoding MB 0 8, bytestream 112456 [h264 @ 0x55bb51b75980] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55bb51b75980] error while decoding MB 0 23, bytestream 236 [h264 @ 0x55bb51c693c0] left block unavailable for requested intra mode [h264 @ 0x55bb51c693c0] error while decoding MB 0 37, bytestream 5308 [h264 @ 0x55bb51b55300] left block unavailable for requested intra m", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 26, "h264_decode_error_count": 38, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T13:00:46+09:00", "last_error_summary": "[h264 @ 0x55bb528a98c0] left block unavailable for requested intra mode [h264 @ 0x55bb528a98c0] error while decoding MB 0 8, bytestream 112456 [h264 @ 0x55bb51b75980] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55bb51b75980] error while decoding MB 0 23, bytestream 236 [h264 @ 0x55bb51c693c0] left block unavailable for requested intra mode [h264 @ 0x55bb51c693c0] error while decoding MB 0 37, bytestream 5308 [h264 @ 0x55bb51b55300] left block unavailable for requested intra m", "last_exit_at": "2026-07-02T12:24:10+09:00", "last_returncode": 0, "last_stable_duration_sec": 2195.579, "last_started_at": "2026-07-02T12:24:10+09:00", "pid": 7955, "process": {"exists": false, "pid": 7955}, "restart_count": 33, "restart_count_per_hour": 11, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 34}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 141.043, "sample_count": 2409, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 104381, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T13:00:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.072, "last_frame_pushed_at": "2026-07-02T13:00:46+09:00", "last_frame_read_age_sec": 0.06, "last_frame_read_at": "2026-07-02T13:00:46+09:00", "mjpeg": {"age_sec": 0.056, "exists": true, "mtime": "2026-07-02T13:00:46.428513+09:00", "mtime_epoch": 1782964846.428513, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199597}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T13:00:46+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1332.766, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 117, "pushed_frames": 117, "read_frames": 117, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.908, "pushed": 3.908, "read": 3.908}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.763, "count": 117, "max_ms": 5.564, "p95_ms": 4.693}, "gstreamer_push": {"avg_ms": 1.276, "count": 117, "max_ms": 1.642, "p95_ms": 1.484}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.716, "count": 117, "max_ms": 70.874, "p95_ms": 65.171}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.63, "count": 117, "max_ms": 98.621, "p95_ms": 85.654}, "yolo11_640_fire_11class": {"avg_ms": 51.392, "count": 117, "max_ms": 77.975, "p95_ms": 69.029}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.465, "count": 117, "max_ms": 88.197, "p95_ms": 79.525}}, "truncated_count": 0}, "processing_total": {"avg_ms": 74.499, "count": 117, "max_ms": 111.334, "p95_ms": 99.112}, "read_interval": {"avg_ms": 255.904, "count": 117, "max_ms": 19394.423, "p95_ms": 116.253}, "triton_wait": {"avg_ms": 62.288, "count": 117, "max_ms": 98.632, "p95_ms": 85.668}, "visualize": {"avg_ms": 3.58, "count": 117, "max_ms": 4.707, "p95_ms": 4.339}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.937, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 869}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5118.723, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T13:00:46+09:00", "last_error_summary": "[h264 @ 0x55bb528a98c0] left block unavailable for requested intra mode [h264 @ 0x55bb528a98c0] error while decoding MB 0 8, bytestream 112456 [h264 @ 0x55bb51b75980] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55bb51b75980] error while decoding MB 0 23, bytestream 236 [h264 @ 0x55bb51c693c0] left block unavailable for requested intra mode [h264 @ 0x55bb51c693c0] error while decoding MB 0 37, bytestream 5308 [h264 @ 0x55bb51b55300] left block unavailable for requested intra m", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 26, "h264_decode_error_count": 38, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T13:00:46+09:00", "last_error_summary": "[h264 @ 0x55bb528a98c0] left block unavailable for requested intra mode [h264 @ 0x55bb528a98c0] error while decoding MB 0 8, bytestream 112456 [h264 @ 0x55bb51b75980] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55bb51b75980] error while decoding MB 0 23, bytestream 236 [h264 @ 0x55bb51c693c0] left block unavailable for requested intra mode [h264 @ 0x55bb51c693c0] error while decoding MB 0 37, bytestream 5308 [h264 @ 0x55bb51b55300] left block unavailable for requested intra m", "last_exit_at": "2026-07-02T13:00:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 2195.609, "last_started_at": "2026-07-02T12:24:10+09:00", "pid": 7955, "process": {"exists": false, "pid": 7955}, "restart_count": 33, "restart_count_per_hour": 11, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 34}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 141.043, "sample_count": 2410, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 104382, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T13:00:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T13:00:46+09:00", "last_frame_read_age_sec": 0.089, "last_frame_read_at": "2026-07-02T13:00:46+09:00", "mjpeg": {"age_sec": 0.086, "exists": true, "mtime": "2026-07-02T13:00:46.428513+09:00", "mtime_epoch": 1782964846.428513, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199597}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T13:00:46+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1332.77, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 118, "pushed_frames": 118, "read_frames": 117, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.938, "pushed": 3.938, "read": 3.904}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.762, "count": 118, "max_ms": 5.564, "p95_ms": 4.693}, "gstreamer_push": {"avg_ms": 1.277, "count": 118, "max_ms": 1.642, "p95_ms": 1.484}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.685, "count": 118, "max_ms": 70.874, "p95_ms": 65.171}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.698, "count": 118, "max_ms": 98.621, "p95_ms": 85.654}, "yolo11_640_fire_11class": {"avg_ms": 51.344, "count": 118, "max_ms": 77.975, "p95_ms": 69.029}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.401, "count": 118, "max_ms": 88.197, "p95_ms": 79.525}}, "truncated_count": 0}, "processing_total": {"avg_ms": 74.553, "count": 118, "max_ms": 111.334, "p95_ms": 99.112}, "read_interval": {"avg_ms": 255.904, "count": 117, "max_ms": 19394.423, "p95_ms": 116.253}, "triton_wait": {"avg_ms": 62.342, "count": 118, "max_ms": 98.632, "p95_ms": 85.668}, "visualize": {"avg_ms": 3.581, "count": 118, "max_ms": 4.707, "p95_ms": 4.339}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.966, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 869}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5118.5, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 26, "h264_decode_error_count": 38, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T13:00:46+09:00", "last_error_summary": "[h264 @ 0x55bb528a98c0] left block unavailable for requested intra mode [h264 @ 0x55bb528a98c0] error while decoding MB 0 8, bytestream 112456 [h264 @ 0x55bb51b75980] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55bb51b75980] error while decoding MB 0 23, bytestream 236 [h264 @ 0x55bb51c693c0] left block unavailable for requested intra mode [h264 @ 0x55bb51c693c0] error while decoding MB 0 37, bytestream 5308 [h264 @ 0x55bb51b55300] left block unavailable for requested intra m", "last_exit_at": "2026-07-02T13:00:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 2195.616, "last_started_at": "2026-07-02T12:24:10+09:00", "pid": 7955, "process": {"exists": false, "pid": 7955}, "restart_count": 33, "restart_count_per_hour": 11, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 34}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 141.043, "sample_count": 2410, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 104382, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T13:00:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-02T13:00:46+09:00", "last_frame_read_age_sec": 0.097, "last_frame_read_at": "2026-07-02T13:00:46+09:00", "mjpeg": {"age_sec": 0.094, "exists": true, "mtime": "2026-07-02T13:00:46.428513+09:00", "mtime_epoch": 1782964846.428513, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199597}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T13:00:46+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1332.77, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 118, "pushed_frames": 118, "read_frames": 117, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.937, "pushed": 3.937, "read": 3.903}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.762, "count": 118, "max_ms": 5.564, "p95_ms": 4.693}, "gstreamer_push": {"avg_ms": 1.277, "count": 118, "max_ms": 1.642, "p95_ms": 1.484}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.685, "count": 118, "max_ms": 70.874, "p95_ms": 65.171}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.698, "count": 118, "max_ms": 98.621, "p95_ms": 85.654}, "yolo11_640_fire_11class": {"avg_ms": 51.344, "count": 118, "max_ms": 77.975, "p95_ms": 69.029}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.401, "count": 118, "max_ms": 88.197, "p95_ms": 79.525}}, "truncated_count": 0}, "processing_total": {"avg_ms": 74.553, "count": 118, "max_ms": 111.334, "p95_ms": 99.112}, "read_interval": {"avg_ms": 255.904, "count": 117, "max_ms": 19394.423, "p95_ms": 116.253}, "triton_wait": {"avg_ms": 62.342, "count": 118, "max_ms": 98.632, "p95_ms": 85.668}, "visualize": {"avg_ms": 3.581, "count": 118, "max_ms": 4.707, "p95_ms": 4.339}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.974, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 869}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5230.09, "limit_mb": 8796093022207.996}, "detail": {"pid": 9154, "restart_count": 34, "start_count": 35}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 26, "h264_decode_error_count": 38, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T13:00:46+09:00", "last_error_summary": "[h264 @ 0x55bb528a98c0] left block unavailable for requested intra mode [h264 @ 0x55bb528a98c0] error while decoding MB 0 8, bytestream 112456 [h264 @ 0x55bb51b75980] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55bb51b75980] error while decoding MB 0 23, bytestream 236 [h264 @ 0x55bb51c693c0] left block unavailable for requested intra mode [h264 @ 0x55bb51c693c0] error while decoding MB 0 37, bytestream 5308 [h264 @ 0x55bb51b55300] left block unavailable for requested intra m", "last_exit_at": "2026-07-02T13:00:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T13:00:49+09:00", "pid": 9154, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.512, "name": "ffmpeg", "pid": 9154, "rss_mb": 136.512, "state": "S (sleeping)", "threads": 106}, "restart_count": 34, "restart_count_per_hour": 12, "rtsp_timeout_count": 8, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 35}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 141.083, "sample_count": 2360, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 104382, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T13:00:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.922, "last_frame_pushed_at": "2026-07-02T13:00:46+09:00", "last_frame_read_age_sec": 5.005, "last_frame_read_at": "2026-07-02T13:00:46+09:00", "mjpeg": {"age_sec": 4.905, "exists": true, "mtime": "2026-07-02T13:00:46.524515+09:00", "mtime_epoch": 1782964846.5245152, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200502}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T13:00:51+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1332.77, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 63, "pushed_frames": 63, "read_frames": 62, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.102, "pushed": 2.102, "read": 2.069}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.792, "count": 63, "max_ms": 4.684, "p95_ms": 4.39}, "gstreamer_push": {"avg_ms": 1.305, "count": 63, "max_ms": 1.642, "p95_ms": 1.484}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.948, "count": 63, "max_ms": 70.874, "p95_ms": 65.171}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.157, "count": 63, "max_ms": 98.621, "p95_ms": 83.592}, "yolo11_640_fire_11class": {"avg_ms": 52.044, "count": 63, "max_ms": 77.975, "p95_ms": 71.373}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.001, "count": 63, "max_ms": 88.197, "p95_ms": 79.525}}, "truncated_count": 0}, "processing_total": {"avg_ms": 75.885, "count": 63, "max_ms": 111.334, "p95_ms": 96.579}, "read_interval": {"avg_ms": 403.723, "count": 62, "max_ms": 19394.423, "p95_ms": 116.253}, "triton_wait": {"avg_ms": 63.463, "count": 63, "max_ms": 98.632, "p95_ms": 83.605}, "visualize": {"avg_ms": 3.668, "count": 63, "max_ms": 4.565, "p95_ms": 4.101}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.972, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 32, "tracking_summary": 877}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5066.215, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55be7460fb40] error while decoding MB 52 21, bytestream -37 [in#0/rtsp @ 0x55be744b9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55be744b9e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 21, "h264_decode_error_count": 21, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T13:02:06+09:00", "last_error_summary": "[h264 @ 0x55be7460fb40] error while decoding MB 52 21, bytestream -37 [in#0/rtsp @ 0x55be744b9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55be744b9e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T12:59:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 164.62, "last_started_at": "2026-07-02T12:59:22+09:00", "pid": 9002, "process": {"exists": false, "pid": 9002}, "restart_count": 20, "restart_count_per_hour": 7, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 21}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.597, "sample_count": 1900, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 105326, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T13:02:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.07, "last_frame_pushed_at": "2026-07-02T13:02:06+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-02T13:02:06+09:00", "mjpeg": {"age_sec": 0.052, "exists": true, "mtime": "2026-07-02T13:02:06.578162+09:00", "mtime_epoch": 1782964926.5781624, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 249753}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T13:02:06+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1081.008, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.681, "pushed": 7.681, "read": 8.321}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.419, "count": 12, "max_ms": 8.537, "p95_ms": 8.537}, "gstreamer_push": {"avg_ms": 1.153, "count": 12, "max_ms": 1.283, "p95_ms": 1.283}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.534, "count": 12, "max_ms": 71.009, "p95_ms": 71.009}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.61, "count": 12, "max_ms": 98.155, "p95_ms": 98.155}, "yolo11_640_fire_11class": {"avg_ms": 54.809, "count": 12, "max_ms": 70.646, "p95_ms": 70.646}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.673, "count": 12, "max_ms": 78.571, "p95_ms": 78.571}}, "truncated_count": 0}, "processing_total": {"avg_ms": 112.859, "count": 12, "max_ms": 144.355, "p95_ms": 144.355}, "read_interval": {"avg_ms": 2311.632, "count": 13, "max_ms": 28550.653, "p95_ms": 28550.653}, "triton_wait": {"avg_ms": 63.596, "count": 12, "max_ms": 98.164, "p95_ms": 98.164}, "visualize": {"avg_ms": 4.907, "count": 12, "max_ms": 5.964, "p95_ms": 5.964}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 1.562, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 32, "tracking_summary": 877}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5056.914, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T13:02:06+09:00", "last_error_summary": "[h264 @ 0x55be7460fb40] error while decoding MB 52 21, bytestream -37 [in#0/rtsp @ 0x55be744b9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55be744b9e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 21, "h264_decode_error_count": 21, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T13:02:06+09:00", "last_error_summary": "[h264 @ 0x55be7460fb40] error while decoding MB 52 21, bytestream -37 [in#0/rtsp @ 0x55be744b9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55be744b9e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T13:02:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 164.689, "last_started_at": "2026-07-02T12:59:22+09:00", "pid": 9002, "process": {"exists": false, "pid": 9002}, "restart_count": 20, "restart_count_per_hour": 7, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 21}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.597, "sample_count": 1901, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 105327, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T13:02:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T13:02:06+09:00", "last_frame_read_age_sec": 0.13, "last_frame_read_at": "2026-07-02T13:02:06+09:00", "mjpeg": {"age_sec": 0.121, "exists": true, "mtime": "2026-07-02T13:02:06.578162+09:00", "mtime_epoch": 1782964926.5781624, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 249753}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T13:02:06+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1081.008, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.97, "pushed": 7.97, "read": 7.97}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.472, "count": 13, "max_ms": 8.537, "p95_ms": 8.537}, "gstreamer_push": {"avg_ms": 1.146, "count": 13, "max_ms": 1.283, "p95_ms": 1.283}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.774, "count": 13, "max_ms": 71.009, "p95_ms": 71.009}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.547, "count": 13, "max_ms": 98.155, "p95_ms": 98.155}, "yolo11_640_fire_11class": {"avg_ms": 55.519, "count": 13, "max_ms": 70.646, "p95_ms": 70.646}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.021, "count": 13, "max_ms": 78.571, "p95_ms": 78.571}}, "truncated_count": 0}, "processing_total": {"avg_ms": 113.797, "count": 13, "max_ms": 144.355, "p95_ms": 144.355}, "read_interval": {"avg_ms": 2311.632, "count": 13, "max_ms": 28550.653, "p95_ms": 28550.653}, "triton_wait": {"avg_ms": 64.072, "count": 13, "max_ms": 98.164, "p95_ms": 98.164}, "visualize": {"avg_ms": 4.888, "count": 13, "max_ms": 5.964, "p95_ms": 5.964}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 1.631, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 32, "tracking_summary": 877}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5056.906, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 21, "h264_decode_error_count": 21, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T13:02:06+09:00", "last_error_summary": "[h264 @ 0x55be7460fb40] error while decoding MB 52 21, bytestream -37 [in#0/rtsp @ 0x55be744b9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55be744b9e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T13:02:06+09:00", "last_returncode": 0, "last_stable_duration_sec": 164.693, "last_started_at": "2026-07-02T12:59:22+09:00", "pid": 9002, "process": {"exists": false, "pid": 9002}, "restart_count": 20, "restart_count_per_hour": 7, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 21}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.597, "sample_count": 1901, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 105327, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T13:02:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-02T13:02:06+09:00", "last_frame_read_age_sec": 0.135, "last_frame_read_at": "2026-07-02T13:02:06+09:00", "mjpeg": {"age_sec": 0.126, "exists": true, "mtime": "2026-07-02T13:02:06.578162+09:00", "mtime_epoch": 1782964926.5781624, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 249753}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T13:02:06+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1081.008, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.949, "pushed": 7.949, "read": 7.949}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.472, "count": 13, "max_ms": 8.537, "p95_ms": 8.537}, "gstreamer_push": {"avg_ms": 1.146, "count": 13, "max_ms": 1.283, "p95_ms": 1.283}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.774, "count": 13, "max_ms": 71.009, "p95_ms": 71.009}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.547, "count": 13, "max_ms": 98.155, "p95_ms": 98.155}, "yolo11_640_fire_11class": {"avg_ms": 55.519, "count": 13, "max_ms": 70.646, "p95_ms": 70.646}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.021, "count": 13, "max_ms": 78.571, "p95_ms": 78.571}}, "truncated_count": 0}, "processing_total": {"avg_ms": 113.797, "count": 13, "max_ms": 144.355, "p95_ms": 144.355}, "read_interval": {"avg_ms": 2311.632, "count": 13, "max_ms": 28550.653, "p95_ms": 28550.653}, "triton_wait": {"avg_ms": 64.072, "count": 13, "max_ms": 98.164, "p95_ms": 98.164}, "visualize": {"avg_ms": 4.888, "count": 13, "max_ms": 5.964, "p95_ms": 5.964}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 1.635, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 32, "tracking_summary": 877}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5095.637, "limit_mb": 8796093022207.996}, "detail": {"pid": 9261, "restart_count": 21, "start_count": 22}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 21, "h264_decode_error_count": 21, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T13:02:06+09:00", "last_error_summary": "[h264 @ 0x55be7460fb40] error while decoding MB 52 21, bytestream -37 [in#0/rtsp @ 0x55be744b9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55be744b9e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T13:02:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T13:02:09+09:00", "pid": 9261, "process": {"exists": true, "fd_count": 5, "hwm_mb": 65.906, "name": "ffmpeg", "pid": 9261, "rss_mb": 65.906, "state": "S (sleeping)", "threads": 19}, "restart_count": 21, "restart_count_per_hour": 8, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 22}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 148.56, "sample_count": 1868, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 105327, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T13:02:06+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 4.463, "last_frame_pushed_at": "2026-07-02T13:02:06+09:00", "last_frame_read_age_sec": 4.589, "last_frame_read_at": "2026-07-02T13:02:06+09:00", "mjpeg": {"age_sec": 4.444, "exists": true, "mtime": "2026-07-02T13:02:06.714165+09:00", "mtime_epoch": 1782964926.7141652, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 241869}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T13:02:11+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1081.008, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.135, "pushed": 2.135, "read": 2.135}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.472, "count": 13, "max_ms": 8.537, "p95_ms": 8.537}, "gstreamer_push": {"avg_ms": 1.146, "count": 13, "max_ms": 1.283, "p95_ms": 1.283}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.774, "count": 13, "max_ms": 71.009, "p95_ms": 71.009}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.547, "count": 13, "max_ms": 98.155, "p95_ms": 98.155}, "yolo11_640_fire_11class": {"avg_ms": 55.519, "count": 13, "max_ms": 70.646, "p95_ms": 70.646}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.021, "count": 13, "max_ms": 78.571, "p95_ms": 78.571}}, "truncated_count": 0}, "processing_total": {"avg_ms": 113.797, "count": 13, "max_ms": 144.355, "p95_ms": 144.355}, "read_interval": {"avg_ms": 2311.632, "count": 13, "max_ms": 28550.653, "p95_ms": 28550.653}, "triton_wait": {"avg_ms": 64.072, "count": 13, "max_ms": 98.164, "p95_ms": 98.164}, "visualize": {"avg_ms": 4.888, "count": 13, "max_ms": 5.964, "p95_ms": 5.964}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 6.09, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 902}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5138.254, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 5, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x563755324140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563755324140] error while decoding MB 0 25, bytestream 3568 [h264 @ 0x563755e64000] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563755e64000] error while decoding MB 0 13, bytestream 4206 [h264 @ 0x563755324140] left block unavailable for requested intra mode [h264 @ 0x563755324140] error while decoding MB 0 29, bytestream 144235 [h264 @ 0x563755e64000] left block unavailable for requested", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 27, "h264_decode_error_count": 43, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T13:08:07+09:00", "last_error_summary": "[h264 @ 0x563755324140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563755324140] error while decoding MB 0 25, bytestream 3568 [h264 @ 0x563755e64000] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563755e64000] error while decoding MB 0 13, bytestream 4206 [h264 @ 0x563755324140] left block unavailable for requested intra mode [h264 @ 0x563755324140] error while decoding MB 0 29, bytestream 144235 [h264 @ 0x563755e64000] left block unavailable for requested", "last_exit_at": "2026-07-02T13:00:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 437.836, "last_started_at": "2026-07-02T13:00:49+09:00", "pid": 9154, "process": {"exists": false, "pid": 9154}, "restart_count": 34, "restart_count_per_hour": 9, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 35}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 141.538, "sample_count": 2572, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 108317, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T13:08:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-02T13:08:07+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-02T13:08:07+09:00", "mjpeg": {"age_sec": 0.037, "exists": true, "mtime": "2026-07-02T13:08:07.227628+09:00", "mtime_epoch": 1782965287.2276285, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190305}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T13:08:07+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1334.574, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 89, "pushed_frames": 89, "read_frames": 89, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.968, "pushed": 2.968, "read": 2.968}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.13, "count": 89, "max_ms": 9.034, "p95_ms": 8.237}, "gstreamer_push": {"avg_ms": 1.12, "count": 89, "max_ms": 1.499, "p95_ms": 1.371}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.793, "count": 89, "max_ms": 68.478, "p95_ms": 62.92}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.906, "count": 89, "max_ms": 105.376, "p95_ms": 87.126}, "yolo11_640_fire_11class": {"avg_ms": 49.478, "count": 89, "max_ms": 75.821, "p95_ms": 64.317}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.622, "count": 89, "max_ms": 95.462, "p95_ms": 86.79}}, "truncated_count": 0}, "processing_total": {"avg_ms": 112.795, "count": 89, "max_ms": 179.791, "p95_ms": 158.716}, "read_interval": {"avg_ms": 337.325, "count": 89, "max_ms": 10128.174, "p95_ms": 178.85}, "triton_wait": {"avg_ms": 64.503, "count": 89, "max_ms": 105.388, "p95_ms": 89.774}, "visualize": {"avg_ms": 5.004, "count": 89, "max_ms": 7.92, "p95_ms": 6.418}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.984, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 902}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5137.297, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T13:08:07+09:00", "last_error_summary": "[h264 @ 0x563755324140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563755324140] error while decoding MB 0 25, bytestream 3568 [h264 @ 0x563755e64000] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563755e64000] error while decoding MB 0 13, bytestream 4206 [h264 @ 0x563755324140] left block unavailable for requested intra mode [h264 @ 0x563755324140] error while decoding MB 0 29, bytestream 144235 [h264 @ 0x563755e64000] left block unavailable for requested", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 27, "h264_decode_error_count": 43, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T13:08:07+09:00", "last_error_summary": "[h264 @ 0x563755324140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563755324140] error while decoding MB 0 25, bytestream 3568 [h264 @ 0x563755e64000] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563755e64000] error while decoding MB 0 13, bytestream 4206 [h264 @ 0x563755324140] left block unavailable for requested intra mode [h264 @ 0x563755324140] error while decoding MB 0 29, bytestream 144235 [h264 @ 0x563755e64000] left block unavailable for requested", "last_exit_at": "2026-07-02T13:08:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 437.916, "last_started_at": "2026-07-02T13:00:49+09:00", "pid": 9154, "process": {"exists": false, "pid": 9154}, "restart_count": 34, "restart_count_per_hour": 9, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 35}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 141.538, "sample_count": 2573, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 108318, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T13:08:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-02T13:08:07+09:00", "last_frame_read_age_sec": 0.13, "last_frame_read_at": "2026-07-02T13:08:07+09:00", "mjpeg": {"age_sec": 0.117, "exists": true, "mtime": "2026-07-02T13:08:07.227628+09:00", "mtime_epoch": 1782965287.2276285, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190305}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T13:08:07+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1334.574, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 89, "pushed_frames": 89, "read_frames": 88, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.969, "pushed": 2.969, "read": 2.935}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.189, "count": 89, "max_ms": 9.279, "p95_ms": 8.272}, "gstreamer_push": {"avg_ms": 1.117, "count": 89, "max_ms": 1.499, "p95_ms": 1.371}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.714, "count": 89, "max_ms": 68.478, "p95_ms": 62.92}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.605, "count": 89, "max_ms": 105.376, "p95_ms": 87.126}, "yolo11_640_fire_11class": {"avg_ms": 49.383, "count": 89, "max_ms": 75.821, "p95_ms": 64.317}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.555, "count": 89, "max_ms": 95.462, "p95_ms": 86.79}}, "truncated_count": 0}, "processing_total": {"avg_ms": 113.15, "count": 89, "max_ms": 179.791, "p95_ms": 158.716}, "read_interval": {"avg_ms": 339.982, "count": 88, "max_ms": 10128.174, "p95_ms": 178.85}, "triton_wait": {"avg_ms": 64.263, "count": 89, "max_ms": 105.388, "p95_ms": 89.774}, "visualize": {"avg_ms": 5.031, "count": 89, "max_ms": 7.92, "p95_ms": 6.418}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.979, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 902}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5137.285, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 27, "h264_decode_error_count": 43, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T13:08:07+09:00", "last_error_summary": "[h264 @ 0x563755324140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563755324140] error while decoding MB 0 25, bytestream 3568 [h264 @ 0x563755e64000] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563755e64000] error while decoding MB 0 13, bytestream 4206 [h264 @ 0x563755324140] left block unavailable for requested intra mode [h264 @ 0x563755324140] error while decoding MB 0 29, bytestream 144235 [h264 @ 0x563755e64000] left block unavailable for requested", "last_exit_at": "2026-07-02T13:08:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 437.923, "last_started_at": "2026-07-02T13:00:49+09:00", "pid": 9154, "process": {"exists": false, "pid": 9154}, "restart_count": 34, "restart_count_per_hour": 9, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 35}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 141.538, "sample_count": 2573, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 108318, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T13:08:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-02T13:08:07+09:00", "last_frame_read_age_sec": 0.137, "last_frame_read_at": "2026-07-02T13:08:07+09:00", "mjpeg": {"age_sec": 0.124, "exists": true, "mtime": "2026-07-02T13:08:07.227628+09:00", "mtime_epoch": 1782965287.2276285, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190305}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T13:08:07+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1334.574, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 89, "pushed_frames": 89, "read_frames": 88, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.968, "pushed": 2.968, "read": 2.935}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.189, "count": 89, "max_ms": 9.279, "p95_ms": 8.272}, "gstreamer_push": {"avg_ms": 1.117, "count": 89, "max_ms": 1.499, "p95_ms": 1.371}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.714, "count": 89, "max_ms": 68.478, "p95_ms": 62.92}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.605, "count": 89, "max_ms": 105.376, "p95_ms": 87.126}, "yolo11_640_fire_11class": {"avg_ms": 49.383, "count": 89, "max_ms": 75.821, "p95_ms": 64.317}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.555, "count": 89, "max_ms": 95.462, "p95_ms": 86.79}}, "truncated_count": 0}, "processing_total": {"avg_ms": 113.15, "count": 89, "max_ms": 179.791, "p95_ms": 158.716}, "read_interval": {"avg_ms": 339.982, "count": 88, "max_ms": 10128.174, "p95_ms": 178.85}, "triton_wait": {"avg_ms": 64.263, "count": 89, "max_ms": 105.388, "p95_ms": 89.774}, "visualize": {"avg_ms": 5.031, "count": 89, "max_ms": 7.92, "p95_ms": 6.418}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.987, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 902}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5242.742, "limit_mb": 8796093022207.996}, "detail": {"pid": 9413, "restart_count": 35, "start_count": 36}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 27, "h264_decode_error_count": 43, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T13:08:07+09:00", "last_error_summary": "[h264 @ 0x563755324140] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563755324140] error while decoding MB 0 25, bytestream 3568 [h264 @ 0x563755e64000] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x563755e64000] error while decoding MB 0 13, bytestream 4206 [h264 @ 0x563755324140] left block unavailable for requested intra mode [h264 @ 0x563755324140] error while decoding MB 0 29, bytestream 144235 [h264 @ 0x563755e64000] left block unavailable for requested", "last_exit_at": "2026-07-02T13:08:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T13:08:09+09:00", "pid": 9413, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.133, "name": "ffmpeg", "pid": 9413, "rss_mb": 136.133, "state": "S (sleeping)", "threads": 106}, "restart_count": 35, "restart_count_per_hour": 10, "rtsp_timeout_count": 10, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 36}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 142.003, "sample_count": 2529, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 108318, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T13:08:07+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.343, "last_frame_pushed_at": "2026-07-02T13:08:07+09:00", "last_frame_read_age_sec": 4.464, "last_frame_read_at": "2026-07-02T13:08:07+09:00", "mjpeg": {"age_sec": 4.323, "exists": true, "mtime": "2026-07-02T13:08:07.355628+09:00", "mtime_epoch": 1782965287.3556285, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190336}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T13:08:11+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1334.574, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 49, "pushed_frames": 49, "read_frames": 48, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.639, "pushed": 1.639, "read": 1.606}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 7.127, "count": 49, "max_ms": 9.279, "p95_ms": 8.508}, "gstreamer_push": {"avg_ms": 1.041, "count": 49, "max_ms": 1.246, "p95_ms": 1.162}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.36, "count": 49, "max_ms": 68.478, "p95_ms": 60.501}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.707, "count": 49, "max_ms": 100.184, "p95_ms": 82.481}, "yolo11_640_fire_11class": {"avg_ms": 49.112, "count": 49, "max_ms": 75.821, "p95_ms": 69.512}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.654, "count": 49, "max_ms": 95.462, "p95_ms": 88.943}}, "truncated_count": 0}, "processing_total": {"avg_ms": 128.477, "count": 49, "max_ms": 179.791, "p95_ms": 167.012}, "read_interval": {"avg_ms": 532.103, "count": 48, "max_ms": 10128.174, "p95_ms": 191.531}, "triton_wait": {"avg_ms": 64.495, "count": 49, "max_ms": 100.195, "p95_ms": 88.955}, "visualize": {"avg_ms": 5.596, "count": 49, "max_ms": 7.92, "p95_ms": 6.851}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.895, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 32, "tracking_summary": 902}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5078.602, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55ae9f0c4a80] error while decoding MB 88 26, bytestream -15", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 22, "h264_decode_error_count": 22, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T13:10:52+09:00", "last_error_summary": "[h264 @ 0x55ae9f0c4a80] error while decoding MB 88 26, bytestream -15", "last_exit_at": "2026-07-02T13:02:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 523.594, "last_started_at": "2026-07-02T13:02:09+09:00", "pid": 9261, "process": {"exists": false, "pid": 9261}, "restart_count": 21, "restart_count_per_hour": 8, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 22}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 225.903, "sample_count": 1577, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 108297, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T13:10:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.074, "last_frame_pushed_at": "2026-07-02T13:10:52+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-02T13:10:52+09:00", "mjpeg": {"age_sec": 0.06, "exists": true, "mtime": "2026-07-02T13:10:52.691998+09:00", "mtime_epoch": 1782965452.6919985, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 232484}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T13:10:52+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1082.297, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 184, "pushed_frames": 184, "read_frames": 184, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.142, "pushed": 6.142, "read": 6.142}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.481, "count": 183, "max_ms": 14.759, "p95_ms": 11.357}, "gstreamer_push": {"avg_ms": 1.027, "count": 184, "max_ms": 1.596, "p95_ms": 1.18}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.79, "count": 183, "max_ms": 82.949, "p95_ms": 72.809}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.231, "count": 183, "max_ms": 106.308, "p95_ms": 91.623}, "yolo11_640_fire_11class": {"avg_ms": 54.46, "count": 183, "max_ms": 83.541, "p95_ms": 70.318}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.663, "count": 183, "max_ms": 96.426, "p95_ms": 84.39}}, "truncated_count": 0}, "processing_total": {"avg_ms": 152.181, "count": 184, "max_ms": 191.026, "p95_ms": 178.301}, "read_interval": {"avg_ms": 163.363, "count": 184, "max_ms": 203.255, "p95_ms": 191.058}, "triton_wait": {"avg_ms": 66.694, "count": 183, "max_ms": 106.322, "p95_ms": 93.771}, "visualize": {"avg_ms": 5.147, "count": 183, "max_ms": 7.984, "p95_ms": 5.725}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.959, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 32, "tracking_summary": 902}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5077.801, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T13:10:52+09:00", "last_error_summary": "[h264 @ 0x55ae9f0c4a80] error while decoding MB 88 26, bytestream -15", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 22, "h264_decode_error_count": 22, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T13:10:52+09:00", "last_error_summary": "[h264 @ 0x55ae9f0c4a80] error while decoding MB 88 26, bytestream -15", "last_exit_at": "2026-07-02T13:10:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 523.696, "last_started_at": "2026-07-02T13:02:09+09:00", "pid": 9261, "process": {"exists": false, "pid": 9261}, "restart_count": 21, "restart_count_per_hour": 8, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 22}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 225.903, "sample_count": 1578, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 108298, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T13:10:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T13:10:52+09:00", "last_frame_read_age_sec": 0.166, "last_frame_read_at": "2026-07-02T13:10:52+09:00", "mjpeg": {"age_sec": 0.161, "exists": true, "mtime": "2026-07-02T13:10:52.691998+09:00", "mtime_epoch": 1782965452.6919985, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 232484}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T13:10:52+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1082.297, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 184, "pushed_frames": 184, "read_frames": 183, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.153, "pushed": 6.153, "read": 6.12}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.511, "count": 184, "max_ms": 14.976, "p95_ms": 11.642}, "gstreamer_push": {"avg_ms": 1.027, "count": 184, "max_ms": 1.596, "p95_ms": 1.18}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.716, "count": 184, "max_ms": 82.949, "p95_ms": 72.809}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.233, "count": 184, "max_ms": 106.308, "p95_ms": 91.623}, "yolo11_640_fire_11class": {"avg_ms": 54.425, "count": 184, "max_ms": 83.541, "p95_ms": 70.318}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.651, "count": 184, "max_ms": 96.426, "p95_ms": 84.39}}, "truncated_count": 0}, "processing_total": {"avg_ms": 152.143, "count": 184, "max_ms": 191.026, "p95_ms": 178.301}, "read_interval": {"avg_ms": 163.287, "count": 183, "max_ms": 203.255, "p95_ms": 191.058}, "triton_wait": {"avg_ms": 66.676, "count": 184, "max_ms": 106.322, "p95_ms": 93.771}, "visualize": {"avg_ms": 5.16, "count": 184, "max_ms": 7.984, "p95_ms": 5.909}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.904, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 32, "tracking_summary": 902}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5077.855, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 22, "h264_decode_error_count": 22, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T13:10:52+09:00", "last_error_summary": "[h264 @ 0x55ae9f0c4a80] error while decoding MB 88 26, bytestream -15", "last_exit_at": "2026-07-02T13:10:52+09:00", "last_returncode": 0, "last_stable_duration_sec": 523.7, "last_started_at": "2026-07-02T13:02:09+09:00", "pid": 9261, "process": {"exists": false, "pid": 9261}, "restart_count": 21, "restart_count_per_hour": 8, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": false, "start_count": 22}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 225.903, "sample_count": 1578, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 108298, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T13:10:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-02T13:10:52+09:00", "last_frame_read_age_sec": 0.17, "last_frame_read_at": "2026-07-02T13:10:52+09:00", "mjpeg": {"age_sec": 0.165, "exists": true, "mtime": "2026-07-02T13:10:52.691998+09:00", "mtime_epoch": 1782965452.6919985, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 232484}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T13:10:52+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1082.297, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 184, "pushed_frames": 184, "read_frames": 183, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.152, "pushed": 6.152, "read": 6.119}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.511, "count": 184, "max_ms": 14.976, "p95_ms": 11.642}, "gstreamer_push": {"avg_ms": 1.027, "count": 184, "max_ms": 1.596, "p95_ms": 1.18}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.716, "count": 184, "max_ms": 82.949, "p95_ms": 72.809}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.233, "count": 184, "max_ms": 106.308, "p95_ms": 91.623}, "yolo11_640_fire_11class": {"avg_ms": 54.425, "count": 184, "max_ms": 83.541, "p95_ms": 70.318}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.651, "count": 184, "max_ms": 96.426, "p95_ms": 84.39}}, "truncated_count": 0}, "processing_total": {"avg_ms": 152.143, "count": 184, "max_ms": 191.026, "p95_ms": 178.301}, "read_interval": {"avg_ms": 163.287, "count": 183, "max_ms": 203.255, "p95_ms": 191.058}, "triton_wait": {"avg_ms": 66.676, "count": 184, "max_ms": 106.322, "p95_ms": 93.771}, "visualize": {"avg_ms": 5.16, "count": 184, "max_ms": 7.984, "p95_ms": 5.909}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.908, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 32, "tracking_summary": 902}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5242.562, "limit_mb": 8796093022207.996}, "detail": {"pid": 9520, "restart_count": 22, "start_count": 23}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 22, "h264_decode_error_count": 22, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T13:10:52+09:00", "last_error_summary": "[h264 @ 0x55ae9f0c4a80] error while decoding MB 88 26, bytestream -15", "last_exit_at": "2026-07-02T13:10:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T13:10:55+09:00", "pid": 9520, "process": {"exists": true, "fd_count": 5, "hwm_mb": 194.949, "name": "ffmpeg", "pid": 9520, "rss_mb": 194.949, "state": "S (sleeping)", "threads": 106}, "restart_count": 22, "restart_count_per_hour": 9, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 12, "rtsp_tolerant_mode": false, "running": true, "start_count": 23}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 225.934, "sample_count": 1553, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 108298, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T13:10:52+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 4.963, "last_frame_pushed_at": "2026-07-02T13:10:52+09:00", "last_frame_read_age_sec": 5.124, "last_frame_read_at": "2026-07-02T13:10:52+09:00", "mjpeg": {"age_sec": 4.947, "exists": true, "mtime": "2026-07-02T13:10:52.863999+09:00", "mtime_epoch": 1782965452.8639994, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 231714}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T13:10:57+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1082.297, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 154, "pushed_frames": 154, "read_frames": 153, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.155, "pushed": 5.155, "read": 5.122}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.565, "count": 154, "max_ms": 14.976, "p95_ms": 11.65}, "gstreamer_push": {"avg_ms": 1.028, "count": 154, "max_ms": 1.482, "p95_ms": 1.18}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.338, "count": 154, "max_ms": 82.949, "p95_ms": 73.825}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.367, "count": 154, "max_ms": 106.308, "p95_ms": 89.23}, "yolo11_640_fire_11class": {"avg_ms": 53.697, "count": 154, "max_ms": 83.541, "p95_ms": 69.907}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.183, "count": 154, "max_ms": 96.426, "p95_ms": 85.916}}, "truncated_count": 0}, "processing_total": {"avg_ms": 151.543, "count": 154, "max_ms": 191.026, "p95_ms": 177.401}, "read_interval": {"avg_ms": 162.674, "count": 153, "max_ms": 203.255, "p95_ms": 189.04}, "triton_wait": {"avg_ms": 65.913, "count": 154, "max_ms": 106.322, "p95_ms": 94.096}, "visualize": {"avg_ms": 5.192, "count": 154, "max_ms": 7.984, "p95_ms": 5.919}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.873, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 1016}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5163.008, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x561606001280] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561606001280] error while decoding MB 0 34, bytestream 56920 [h264 @ 0x56160547a300] cabac decode of qscale diff failed at 43 29 [h264 @ 0x56160547a300] error while decoding MB 43 29, bytestream 135 [in#0/rtsp @ 0x56160530de00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56160530de00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 28, "h264_decode_error_count": 46, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T13:31:46+09:00", "last_error_summary": "[h264 @ 0x561606001280] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561606001280] error while decoding MB 0 34, bytestream 56920 [h264 @ 0x56160547a300] cabac decode of qscale diff failed at 43 29 [h264 @ 0x56160547a300] error while decoding MB 43 29, bytestream 135 [in#0/rtsp @ 0x56160530de00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56160530de00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T13:08:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 1417.292, "last_started_at": "2026-07-02T13:08:09+09:00", "pid": 9413, "process": {"exists": false, "pid": 9413}, "restart_count": 35, "restart_count_per_hour": 2, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 36}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.203, "sample_count": 2797, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 121989, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T13:31:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.072, "last_frame_pushed_at": "2026-07-02T13:31:46+09:00", "last_frame_read_age_sec": 0.059, "last_frame_read_at": "2026-07-02T13:31:46+09:00", "mjpeg": {"age_sec": 0.055, "exists": true, "mtime": "2026-07-02T13:31:46.915291+09:00", "mtime_epoch": 1782966706.9152913, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186040}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T13:31:46+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1341.02, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 110, "pushed_frames": 110, "read_frames": 110, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.67, "pushed": 3.67, "read": 3.67}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.98, "count": 110, "max_ms": 5.759, "p95_ms": 4.79}, "gstreamer_push": {"avg_ms": 1.34, "count": 110, "max_ms": 2.169, "p95_ms": 1.506}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.642, "count": 110, "max_ms": 78.285, "p95_ms": 75.451}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.95, "count": 110, "max_ms": 108.558, "p95_ms": 93.293}, "yolo11_640_fire_11class": {"avg_ms": 55.291, "count": 110, "max_ms": 81.846, "p95_ms": 75.937}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.169, "count": 110, "max_ms": 95.539, "p95_ms": 87.286}}, "truncated_count": 0}, "processing_total": {"avg_ms": 81.165, "count": 110, "max_ms": 121.702, "p95_ms": 106.963}, "read_interval": {"avg_ms": 272.512, "count": 110, "max_ms": 19280.514, "p95_ms": 125.658}, "triton_wait": {"avg_ms": 68.327, "count": 110, "max_ms": 108.571, "p95_ms": 94.492}, "visualize": {"avg_ms": 3.724, "count": 110, "max_ms": 4.994, "p95_ms": 4.124}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.969, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 1016}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5162.668, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T13:31:46+09:00", "last_error_summary": "[h264 @ 0x561606001280] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561606001280] error while decoding MB 0 34, bytestream 56920 [h264 @ 0x56160547a300] cabac decode of qscale diff failed at 43 29 [h264 @ 0x56160547a300] error while decoding MB 43 29, bytestream 135 [in#0/rtsp @ 0x56160530de00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56160530de00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 28, "h264_decode_error_count": 46, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T13:31:46+09:00", "last_error_summary": "[h264 @ 0x561606001280] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561606001280] error while decoding MB 0 34, bytestream 56920 [h264 @ 0x56160547a300] cabac decode of qscale diff failed at 43 29 [h264 @ 0x56160547a300] error while decoding MB 43 29, bytestream 135 [in#0/rtsp @ 0x56160530de00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56160530de00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T13:31:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 1417.318, "last_started_at": "2026-07-02T13:08:09+09:00", "pid": 9413, "process": {"exists": false, "pid": 9413}, "restart_count": 35, "restart_count_per_hour": 2, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 36}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.203, "sample_count": 2797, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 121990, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T13:31:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-02T13:31:46+09:00", "last_frame_read_age_sec": 0.086, "last_frame_read_at": "2026-07-02T13:31:46+09:00", "mjpeg": {"age_sec": 0.081, "exists": true, "mtime": "2026-07-02T13:31:46.915291+09:00", "mtime_epoch": 1782966706.9152913, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 186040}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T13:31:47+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1341.02, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 110, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.701, "pushed": 3.701, "read": 3.667}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.985, "count": 111, "max_ms": 5.759, "p95_ms": 4.79}, "gstreamer_push": {"avg_ms": 1.341, "count": 111, "max_ms": 2.169, "p95_ms": 1.506}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.576, "count": 111, "max_ms": 78.285, "p95_ms": 75.451}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.836, "count": 111, "max_ms": 108.558, "p95_ms": 93.293}, "yolo11_640_fire_11class": {"avg_ms": 55.226, "count": 111, "max_ms": 81.846, "p95_ms": 75.937}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.076, "count": 111, "max_ms": 95.539, "p95_ms": 87.286}}, "truncated_count": 0}, "processing_total": {"avg_ms": 81.082, "count": 111, "max_ms": 121.702, "p95_ms": 106.963}, "read_interval": {"avg_ms": 272.512, "count": 110, "max_ms": 19280.514, "p95_ms": 125.658}, "triton_wait": {"avg_ms": 68.2, "count": 111, "max_ms": 108.571, "p95_ms": 94.492}, "visualize": {"avg_ms": 3.762, "count": 111, "max_ms": 7.979, "p95_ms": 4.195}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 1016}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5162.355, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 28, "h264_decode_error_count": 46, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T13:31:46+09:00", "last_error_summary": "[h264 @ 0x561606001280] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561606001280] error while decoding MB 0 34, bytestream 56920 [h264 @ 0x56160547a300] cabac decode of qscale diff failed at 43 29 [h264 @ 0x56160547a300] error while decoding MB 43 29, bytestream 135 [in#0/rtsp @ 0x56160530de00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56160530de00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T13:31:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 1417.326, "last_started_at": "2026-07-02T13:08:09+09:00", "pid": 9413, "process": {"exists": false, "pid": 9413}, "restart_count": 35, "restart_count_per_hour": 2, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 36}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.203, "sample_count": 2797, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 121990, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T13:31:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.02, "last_frame_pushed_at": "2026-07-02T13:31:46+09:00", "last_frame_read_age_sec": 0.094, "last_frame_read_at": "2026-07-02T13:31:46+09:00", "mjpeg": {"age_sec": 0.005, "exists": true, "mtime": "2026-07-02T13:31:46.999293+09:00", "mtime_epoch": 1782966706.9992926, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185747}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T13:31:47+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1341.02, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 110, "pushed_frames": 110, "read_frames": 110, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.668, "pushed": 3.668, "read": 3.668}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.987, "count": 110, "max_ms": 5.759, "p95_ms": 4.79}, "gstreamer_push": {"avg_ms": 1.342, "count": 110, "max_ms": 2.169, "p95_ms": 1.506}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.631, "count": 110, "max_ms": 78.285, "p95_ms": 75.451}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.942, "count": 110, "max_ms": 108.558, "p95_ms": 93.293}, "yolo11_640_fire_11class": {"avg_ms": 55.299, "count": 110, "max_ms": 81.846, "p95_ms": 75.937}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.176, "count": 110, "max_ms": 95.539, "p95_ms": 87.286}}, "truncated_count": 0}, "processing_total": {"avg_ms": 81.207, "count": 110, "max_ms": 121.702, "p95_ms": 106.963}, "read_interval": {"avg_ms": 272.512, "count": 110, "max_ms": 19280.514, "p95_ms": 125.658}, "triton_wait": {"avg_ms": 68.319, "count": 110, "max_ms": 108.571, "p95_ms": 94.492}, "visualize": {"avg_ms": 3.764, "count": 110, "max_ms": 7.979, "p95_ms": 4.195}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.987, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 1016}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5265.246, "limit_mb": 8796093022207.996}, "detail": {"pid": 9672, "restart_count": 36, "start_count": 37}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 28, "h264_decode_error_count": 46, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T13:31:46+09:00", "last_error_summary": "[h264 @ 0x561606001280] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x561606001280] error while decoding MB 0 34, bytestream 56920 [h264 @ 0x56160547a300] cabac decode of qscale diff failed at 43 29 [h264 @ 0x56160547a300] error while decoding MB 43 29, bytestream 135 [in#0/rtsp @ 0x56160530de00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56160530de00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T13:31:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T13:31:49+09:00", "pid": 9672, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.016, "name": "ffmpeg", "pid": 9672, "rss_mb": 136.016, "state": "S (sleeping)", "threads": 106}, "restart_count": 36, "restart_count_per_hour": 3, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 37}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.192, "sample_count": 2748, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 121990, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T13:31:46+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.946, "last_frame_pushed_at": "2026-07-02T13:31:46+09:00", "last_frame_read_age_sec": 5.02, "last_frame_read_at": "2026-07-02T13:31:46+09:00", "mjpeg": {"age_sec": 4.931, "exists": true, "mtime": "2026-07-02T13:31:46.999293+09:00", "mtime_epoch": 1782966706.9992926, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185747}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T13:31:51+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1341.02, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 60, "pushed_frames": 60, "read_frames": 59, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.004, "pushed": 2.004, "read": 1.971}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.029, "count": 60, "max_ms": 5.222, "p95_ms": 4.684}, "gstreamer_push": {"avg_ms": 1.344, "count": 60, "max_ms": 1.58, "p95_ms": 1.493}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.462, "count": 60, "max_ms": 78.285, "p95_ms": 75.451}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 66.062, "count": 60, "max_ms": 103.141, "p95_ms": 95.527}, "yolo11_640_fire_11class": {"avg_ms": 56.452, "count": 60, "max_ms": 81.846, "p95_ms": 75.937}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.099, "count": 60, "max_ms": 95.539, "p95_ms": 91.736}}, "truncated_count": 0}, "processing_total": {"avg_ms": 81.367, "count": 60, "max_ms": 116.086, "p95_ms": 108.612}, "read_interval": {"avg_ms": 423.597, "count": 59, "max_ms": 19280.514, "p95_ms": 128.997}, "triton_wait": {"avg_ms": 68.378, "count": 60, "max_ms": 103.153, "p95_ms": 95.55}, "visualize": {"avg_ms": 3.835, "count": 60, "max_ms": 7.979, "p95_ms": 4.195}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.937, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 32, "tracking_summary": 1034}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5108.879, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x562bc127fe40] error while decoding MB 22 38, bytestream -11 [in#0/rtsp @ 0x562bc1149e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562bc1149e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 23, "h264_decode_error_count": 23, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T13:38:08+09:00", "last_error_summary": "[h264 @ 0x562bc127fe40] error while decoding MB 22 38, bytestream -11 [in#0/rtsp @ 0x562bc1149e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562bc1149e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T13:10:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 1633.1, "last_started_at": "2026-07-02T13:10:55+09:00", "pid": 9520, "process": {"exists": false, "pid": 9520}, "restart_count": 22, "restart_count_per_hour": 8, "rtsp_timeout_count": 30, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 23}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 129.967, "sample_count": 2562, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 124167, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T13:38:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-02T13:38:08+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-02T13:38:08+09:00", "mjpeg": {"age_sec": 0.06, "exists": true, "mtime": "2026-07-02T13:38:08.849417+09:00", "mtime_epoch": 1782967088.8494167, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 222822}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T13:38:08+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1090.215, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.672, "pushed": 0.672, "read": 0.728}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.87, "count": 13, "max_ms": 5.318, "p95_ms": 5.318}, "gstreamer_push": {"avg_ms": 1.287, "count": 12, "max_ms": 1.44, "p95_ms": 1.44}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.22, "count": 13, "max_ms": 66.023, "p95_ms": 66.023}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.949, "count": 13, "max_ms": 90.568, "p95_ms": 90.568}, "yolo11_640_fire_11class": {"avg_ms": 55.542, "count": 13, "max_ms": 66.721, "p95_ms": 66.721}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.994, "count": 13, "max_ms": 82.619, "p95_ms": 82.619}}, "truncated_count": 0}, "processing_total": {"avg_ms": 83.281, "count": 12, "max_ms": 104.506, "p95_ms": 104.506}, "read_interval": {"avg_ms": 2717.544, "count": 13, "max_ms": 17525.895, "p95_ms": 17525.895}, "triton_wait": {"avg_ms": 68.5, "count": 13, "max_ms": 90.58, "p95_ms": 90.58}, "visualize": {"avg_ms": 3.622, "count": 13, "max_ms": 3.813, "p95_ms": 3.813}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.867, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 32, "tracking_summary": 1034}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5099.926, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T13:38:08+09:00", "last_error_summary": "[h264 @ 0x562bc127fe40] error while decoding MB 22 38, bytestream -11 [in#0/rtsp @ 0x562bc1149e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562bc1149e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 23, "h264_decode_error_count": 23, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T13:38:08+09:00", "last_error_summary": "[h264 @ 0x562bc127fe40] error while decoding MB 22 38, bytestream -11 [in#0/rtsp @ 0x562bc1149e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562bc1149e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T13:38:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 1633.12, "last_started_at": "2026-07-02T13:10:55+09:00", "pid": 9520, "process": {"exists": false, "pid": 9520}, "restart_count": 22, "restart_count_per_hour": 8, "rtsp_timeout_count": 30, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 23}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 129.967, "sample_count": 2562, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 124167, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T13:38:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-02T13:38:08+09:00", "last_frame_read_age_sec": 0.085, "last_frame_read_at": "2026-07-02T13:38:08+09:00", "mjpeg": {"age_sec": 0.08, "exists": true, "mtime": "2026-07-02T13:38:08.849417+09:00", "mtime_epoch": 1782967088.8494167, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 222822}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T13:38:08+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1090.215, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.727, "pushed": 0.727, "read": 0.727}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.87, "count": 13, "max_ms": 5.318, "p95_ms": 5.318}, "gstreamer_push": {"avg_ms": 1.292, "count": 13, "max_ms": 1.44, "p95_ms": 1.44}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.22, "count": 13, "max_ms": 66.023, "p95_ms": 66.023}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.949, "count": 13, "max_ms": 90.568, "p95_ms": 90.568}, "yolo11_640_fire_11class": {"avg_ms": 55.542, "count": 13, "max_ms": 66.721, "p95_ms": 66.721}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.994, "count": 13, "max_ms": 82.619, "p95_ms": 82.619}}, "truncated_count": 0}, "processing_total": {"avg_ms": 82.127, "count": 13, "max_ms": 104.506, "p95_ms": 104.506}, "read_interval": {"avg_ms": 2717.544, "count": 13, "max_ms": 17525.895, "p95_ms": 17525.895}, "triton_wait": {"avg_ms": 68.5, "count": 13, "max_ms": 90.58, "p95_ms": 90.58}, "visualize": {"avg_ms": 3.622, "count": 13, "max_ms": 3.813, "p95_ms": 3.813}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.888, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 32, "tracking_summary": 1034}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5099.586, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 23, "h264_decode_error_count": 23, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T13:38:08+09:00", "last_error_summary": "[h264 @ 0x562bc127fe40] error while decoding MB 22 38, bytestream -11 [in#0/rtsp @ 0x562bc1149e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562bc1149e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T13:38:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 1633.128, "last_started_at": "2026-07-02T13:10:55+09:00", "pid": 9520, "process": {"exists": false, "pid": 9520}, "restart_count": 22, "restart_count_per_hour": 8, "rtsp_timeout_count": 30, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": false, "start_count": 23}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 129.967, "sample_count": 2562, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 124167, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T13:38:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.022, "last_frame_pushed_at": "2026-07-02T13:38:08+09:00", "last_frame_read_age_sec": 0.093, "last_frame_read_at": "2026-07-02T13:38:08+09:00", "mjpeg": {"age_sec": 0.004, "exists": true, "mtime": "2026-07-02T13:38:08.933418+09:00", "mtime_epoch": 1782967088.933418, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 222488}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T13:38:08+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1090.215, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.726, "pushed": 0.726, "read": 0.726}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.87, "count": 13, "max_ms": 5.318, "p95_ms": 5.318}, "gstreamer_push": {"avg_ms": 1.292, "count": 13, "max_ms": 1.44, "p95_ms": 1.44}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.22, "count": 13, "max_ms": 66.023, "p95_ms": 66.023}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.949, "count": 13, "max_ms": 90.568, "p95_ms": 90.568}, "yolo11_640_fire_11class": {"avg_ms": 55.542, "count": 13, "max_ms": 66.721, "p95_ms": 66.721}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.994, "count": 13, "max_ms": 82.619, "p95_ms": 82.619}}, "truncated_count": 0}, "processing_total": {"avg_ms": 82.127, "count": 13, "max_ms": 104.506, "p95_ms": 104.506}, "read_interval": {"avg_ms": 2717.544, "count": 13, "max_ms": 17525.895, "p95_ms": 17525.895}, "triton_wait": {"avg_ms": 68.5, "count": 13, "max_ms": 90.58, "p95_ms": 90.58}, "visualize": {"avg_ms": 3.622, "count": 13, "max_ms": 3.813, "p95_ms": 3.813}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.895, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 32, "tracking_summary": 1034}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5121.59, "limit_mb": 8796093022207.996}, "detail": {"pid": 9779, "restart_count": 23, "start_count": 24}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 23, "h264_decode_error_count": 23, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T13:38:08+09:00", "last_error_summary": "[h264 @ 0x562bc127fe40] error while decoding MB 22 38, bytestream -11 [in#0/rtsp @ 0x562bc1149e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x562bc1149e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T13:38:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T13:38:11+09:00", "pid": 9779, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.23, "name": "ffmpeg", "pid": 9779, "rss_mb": 58.23, "state": "S (sleeping)", "threads": 1}, "restart_count": 23, "restart_count_per_hour": 9, "rtsp_timeout_count": 30, "rtsp_timeout_count_per_hour": 14, "rtsp_tolerant_mode": false, "running": true, "start_count": 24}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 130.098, "sample_count": 2509, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 124167, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T13:38:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 4.895, "last_frame_pushed_at": "2026-07-02T13:38:08+09:00", "last_frame_read_age_sec": 4.965, "last_frame_read_at": "2026-07-02T13:38:08+09:00", "mjpeg": {"age_sec": 4.876, "exists": true, "mtime": "2026-07-02T13:38:08.933418+09:00", "mtime_epoch": 1782967088.933418, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 222488}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T13:38:13+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1090.215, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.571, "pushed": 0.571, "read": 0.571}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.87, "count": 13, "max_ms": 5.318, "p95_ms": 5.318}, "gstreamer_push": {"avg_ms": 1.292, "count": 13, "max_ms": 1.44, "p95_ms": 1.44}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.22, "count": 13, "max_ms": 66.023, "p95_ms": 66.023}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 67.949, "count": 13, "max_ms": 90.568, "p95_ms": 90.568}, "yolo11_640_fire_11class": {"avg_ms": 55.542, "count": 13, "max_ms": 66.721, "p95_ms": 66.721}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.994, "count": 13, "max_ms": 82.619, "p95_ms": 82.619}}, "truncated_count": 0}, "processing_total": {"avg_ms": 82.127, "count": 13, "max_ms": 104.506, "p95_ms": 104.506}, "read_interval": {"avg_ms": 2717.544, "count": 13, "max_ms": 17525.895, "p95_ms": 17525.895}, "triton_wait": {"avg_ms": 68.5, "count": 13, "max_ms": 90.58, "p95_ms": 90.58}, "visualize": {"avg_ms": 3.622, "count": 13, "max_ms": 3.813, "p95_ms": 3.813}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.767, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 1059}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5168.363, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55d875948540] error while decoding MB 13 37, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 29, "h264_decode_error_count": 47, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T13:41:14+09:00", "last_error_summary": "[h264 @ 0x55d875948540] error while decoding MB 13 37, bytestream -5", "last_exit_at": "2026-07-02T13:31:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 564.896, "last_started_at": "2026-07-02T13:31:49+09:00", "pid": 9672, "process": {"exists": false, "pid": 9672}, "restart_count": 36, "restart_count_per_hour": 3, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 37}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 142.25, "sample_count": 2480, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 127091, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T13:41:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.049, "last_frame_pushed_at": "2026-07-02T13:41:14+09:00", "last_frame_read_age_sec": 0.04, "last_frame_read_at": "2026-07-02T13:41:14+09:00", "mjpeg": {"age_sec": 0.032, "exists": true, "mtime": "2026-07-02T13:41:14.792960+09:00", "mtime_epoch": 1782967274.7929602, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192083}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T13:41:14+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1343.555, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 208, "pushed_frames": 208, "read_frames": 208, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.955, "pushed": 6.955, "read": 6.955}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.098, "count": 208, "max_ms": 10.363, "p95_ms": 8.641}, "gstreamer_push": {"avg_ms": 1.047, "count": 208, "max_ms": 1.32, "p95_ms": 1.212}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.844, "count": 208, "max_ms": 81.182, "p95_ms": 71.538}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.313, "count": 208, "max_ms": 102.568, "p95_ms": 92.616}, "yolo11_640_fire_11class": {"avg_ms": 50.242, "count": 208, "max_ms": 78.716, "p95_ms": 69.719}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.291, "count": 208, "max_ms": 102.253, "p95_ms": 90.146}}, "truncated_count": 0}, "processing_total": {"avg_ms": 131.719, "count": 208, "max_ms": 187.474, "p95_ms": 171.458}, "read_interval": {"avg_ms": 144.132, "count": 208, "max_ms": 199.918, "p95_ms": 183.977}, "triton_wait": {"avg_ms": 63.555, "count": 208, "max_ms": 102.581, "p95_ms": 92.882}, "visualize": {"avg_ms": 5.89, "count": 208, "max_ms": 8.017, "p95_ms": 7.079}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.907, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 1059}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5168.203, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T13:41:14+09:00", "last_error_summary": "[h264 @ 0x55d875948540] error while decoding MB 13 37, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 29, "h264_decode_error_count": 47, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T13:41:14+09:00", "last_error_summary": "[h264 @ 0x55d875948540] error while decoding MB 13 37, bytestream -5", "last_exit_at": "2026-07-02T13:41:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 565.006, "last_started_at": "2026-07-02T13:31:49+09:00", "pid": 9672, "process": {"exists": false, "pid": 9672}, "restart_count": 36, "restart_count_per_hour": 3, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 37}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 142.382, "sample_count": 2479, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 127092, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T13:41:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T13:41:14+09:00", "last_frame_read_age_sec": 0.151, "last_frame_read_at": "2026-07-02T13:41:14+09:00", "mjpeg": {"age_sec": 0.142, "exists": true, "mtime": "2026-07-02T13:41:14.792960+09:00", "mtime_epoch": 1782967274.7929602, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192083}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T13:41:14+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1343.555, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 208, "pushed_frames": 208, "read_frames": 207, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.958, "pushed": 6.958, "read": 6.924}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.099, "count": 208, "max_ms": 10.363, "p95_ms": 8.641}, "gstreamer_push": {"avg_ms": 1.046, "count": 208, "max_ms": 1.32, "p95_ms": 1.212}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.876, "count": 208, "max_ms": 81.182, "p95_ms": 71.538}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.309, "count": 208, "max_ms": 102.568, "p95_ms": 92.616}, "yolo11_640_fire_11class": {"avg_ms": 50.25, "count": 208, "max_ms": 78.716, "p95_ms": 69.719}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.357, "count": 208, "max_ms": 102.253, "p95_ms": 90.146}}, "truncated_count": 0}, "processing_total": {"avg_ms": 131.866, "count": 208, "max_ms": 187.474, "p95_ms": 171.458}, "read_interval": {"avg_ms": 144.217, "count": 207, "max_ms": 199.918, "p95_ms": 183.977}, "triton_wait": {"avg_ms": 63.589, "count": 208, "max_ms": 102.581, "p95_ms": 92.882}, "visualize": {"avg_ms": 5.893, "count": 208, "max_ms": 8.017, "p95_ms": 7.079}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.895, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 1059}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5168.32, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 29, "h264_decode_error_count": 47, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T13:41:14+09:00", "last_error_summary": "[h264 @ 0x55d875948540] error while decoding MB 13 37, bytestream -5", "last_exit_at": "2026-07-02T13:41:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 565.011, "last_started_at": "2026-07-02T13:31:49+09:00", "pid": 9672, "process": {"exists": false, "pid": 9672}, "restart_count": 36, "restart_count_per_hour": 3, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 37}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 142.382, "sample_count": 2479, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 127092, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T13:41:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-02T13:41:14+09:00", "last_frame_read_age_sec": 0.156, "last_frame_read_at": "2026-07-02T13:41:14+09:00", "mjpeg": {"age_sec": 0.147, "exists": true, "mtime": "2026-07-02T13:41:14.792960+09:00", "mtime_epoch": 1782967274.7929602, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192083}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T13:41:14+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1343.555, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 208, "pushed_frames": 208, "read_frames": 207, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.957, "pushed": 6.957, "read": 6.923}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.099, "count": 208, "max_ms": 10.363, "p95_ms": 8.641}, "gstreamer_push": {"avg_ms": 1.046, "count": 208, "max_ms": 1.32, "p95_ms": 1.212}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.876, "count": 208, "max_ms": 81.182, "p95_ms": 71.538}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.309, "count": 208, "max_ms": 102.568, "p95_ms": 92.616}, "yolo11_640_fire_11class": {"avg_ms": 50.25, "count": 208, "max_ms": 78.716, "p95_ms": 69.719}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.357, "count": 208, "max_ms": 102.253, "p95_ms": 90.146}}, "truncated_count": 0}, "processing_total": {"avg_ms": 131.866, "count": 208, "max_ms": 187.474, "p95_ms": 171.458}, "read_interval": {"avg_ms": 144.217, "count": 207, "max_ms": 199.918, "p95_ms": 183.977}, "triton_wait": {"avg_ms": 63.589, "count": 208, "max_ms": 102.581, "p95_ms": 92.882}, "visualize": {"avg_ms": 5.893, "count": 208, "max_ms": 8.017, "p95_ms": 7.079}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.9, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 1059}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5275.176, "limit_mb": 8796093022207.996}, "detail": {"pid": 9931, "restart_count": 37, "start_count": 38}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 29, "h264_decode_error_count": 47, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T13:41:14+09:00", "last_error_summary": "[h264 @ 0x55d875948540] error while decoding MB 13 37, bytestream -5", "last_exit_at": "2026-07-02T13:41:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T13:41:17+09:00", "pid": 9931, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.996, "name": "ffmpeg", "pid": 9931, "rss_mb": 138.996, "state": "S (sleeping)", "threads": 106}, "restart_count": 37, "restart_count_per_hour": 4, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 38}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 142.423, "sample_count": 2436, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 127092, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T13:41:14+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.791, "last_frame_pushed_at": "2026-07-02T13:41:14+09:00", "last_frame_read_age_sec": 4.937, "last_frame_read_at": "2026-07-02T13:41:14+09:00", "mjpeg": {"age_sec": 4.773, "exists": true, "mtime": "2026-07-02T13:41:14.948966+09:00", "mtime_epoch": 1782967274.9489655, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192085}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T13:41:19+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1343.555, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 172, "pushed_frames": 172, "read_frames": 171, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.74, "pushed": 5.74, "read": 5.707}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.239, "count": 171, "max_ms": 10.363, "p95_ms": 8.765}, "gstreamer_push": {"avg_ms": 1.034, "count": 172, "max_ms": 1.32, "p95_ms": 1.212}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.525, "count": 171, "max_ms": 78.176, "p95_ms": 71.538}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.311, "count": 171, "max_ms": 101.589, "p95_ms": 92.748}, "yolo11_640_fire_11class": {"avg_ms": 49.942, "count": 171, "max_ms": 78.716, "p95_ms": 70.136}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.319, "count": 171, "max_ms": 102.253, "p95_ms": 90.351}}, "truncated_count": 0}, "processing_total": {"avg_ms": 134.897, "count": 172, "max_ms": 187.474, "p95_ms": 173.029}, "read_interval": {"avg_ms": 147.161, "count": 171, "max_ms": 199.918, "p95_ms": 185.534}, "triton_wait": {"avg_ms": 63.249, "count": 171, "max_ms": 102.265, "p95_ms": 92.882}, "visualize": {"avg_ms": 6.046, "count": 171, "max_ms": 8.017, "p95_ms": 7.276}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.963, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 32, "tracking_summary": 1104}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5141.383, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5629e1ee6340] error while decoding MB 76 31, bytestream -37 [in#0/rtsp @ 0x5629e1d1ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5629e1d1ce00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 24, "h264_decode_error_count": 24, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T13:53:12+09:00", "last_error_summary": "[h264 @ 0x5629e1ee6340] error while decoding MB 76 31, bytestream -37 [in#0/rtsp @ 0x5629e1d1ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5629e1d1ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T13:38:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 900.201, "last_started_at": "2026-07-02T13:38:11+09:00", "pid": 9779, "process": {"exists": false, "pid": 9779}, "restart_count": 23, "restart_count_per_hour": 5, "rtsp_timeout_count": 32, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 24}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 102.236, "sample_count": 3018, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 132587, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T13:53:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.078, "last_frame_pushed_at": "2026-07-02T13:53:11+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-02T13:53:11+09:00", "mjpeg": {"age_sec": 0.061, "exists": true, "mtime": "2026-07-02T13:53:11.948580+09:00", "mtime_epoch": 1782967991.94858, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 245381}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T13:53:12+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1095.414, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 114, "pushed_frames": 114, "read_frames": 115, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.8, "pushed": 3.8, "read": 3.833}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.325, "count": 115, "max_ms": 8.583, "p95_ms": 6.328}, "gstreamer_push": {"avg_ms": 1.315, "count": 114, "max_ms": 1.583, "p95_ms": 1.438}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.256, "count": 115, "max_ms": 76.213, "p95_ms": 67.871}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.418, "count": 115, "max_ms": 112.155, "p95_ms": 90.087}, "yolo11_640_fire_11class": {"avg_ms": 57.258, "count": 115, "max_ms": 80.951, "p95_ms": 75.116}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.975, "count": 115, "max_ms": 86.841, "p95_ms": 79.287}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.188, "count": 114, "max_ms": 125.905, "p95_ms": 103.919}, "read_interval": {"avg_ms": 261.33, "count": 115, "max_ms": 10169.434, "p95_ms": 127.424}, "triton_wait": {"avg_ms": 63.063, "count": 115, "max_ms": 112.167, "p95_ms": 90.1}, "visualize": {"avg_ms": 3.701, "count": 115, "max_ms": 4.74, "p95_ms": 4.085}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 32, "tracking_summary": 1104}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5138.582, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T13:53:12+09:00", "last_error_summary": "[h264 @ 0x5629e1ee6340] error while decoding MB 76 31, bytestream -37 [in#0/rtsp @ 0x5629e1d1ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5629e1d1ce00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 24, "h264_decode_error_count": 24, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T13:53:12+09:00", "last_error_summary": "[h264 @ 0x5629e1ee6340] error while decoding MB 76 31, bytestream -37 [in#0/rtsp @ 0x5629e1d1ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5629e1d1ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T13:53:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 900.225, "last_started_at": "2026-07-02T13:38:11+09:00", "pid": 9779, "process": {"exists": false, "pid": 9779}, "restart_count": 23, "restart_count_per_hour": 5, "rtsp_timeout_count": 32, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 24}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 102.236, "sample_count": 3018, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 132587, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T13:53:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-02T13:53:12+09:00", "last_frame_read_age_sec": 0.089, "last_frame_read_at": "2026-07-02T13:53:11+09:00", "mjpeg": {"age_sec": 0.085, "exists": true, "mtime": "2026-07-02T13:53:11.948580+09:00", "mtime_epoch": 1782967991.94858, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 245381}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T13:53:12+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1095.418, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 115, "pushed_frames": 115, "read_frames": 114, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.841, "pushed": 3.841, "read": 3.808}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.325, "count": 115, "max_ms": 8.583, "p95_ms": 6.328}, "gstreamer_push": {"avg_ms": 1.329, "count": 115, "max_ms": 2.97, "p95_ms": 1.442}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.256, "count": 115, "max_ms": 76.213, "p95_ms": 67.871}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.418, "count": 115, "max_ms": 112.155, "p95_ms": 90.087}, "yolo11_640_fire_11class": {"avg_ms": 57.258, "count": 115, "max_ms": 80.951, "p95_ms": 75.116}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.975, "count": 115, "max_ms": 86.841, "p95_ms": 79.287}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.146, "count": 115, "max_ms": 125.905, "p95_ms": 103.919}, "read_interval": {"avg_ms": 262.569, "count": 114, "max_ms": 10169.434, "p95_ms": 127.424}, "triton_wait": {"avg_ms": 63.063, "count": 115, "max_ms": 112.167, "p95_ms": 90.1}, "visualize": {"avg_ms": 3.701, "count": 115, "max_ms": 4.74, "p95_ms": 4.085}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.939, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 32, "tracking_summary": 1104}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5132.516, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 24, "h264_decode_error_count": 24, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T13:53:12+09:00", "last_error_summary": "[h264 @ 0x5629e1ee6340] error while decoding MB 76 31, bytestream -37 [in#0/rtsp @ 0x5629e1d1ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5629e1d1ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T13:53:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 900.234, "last_started_at": "2026-07-02T13:38:11+09:00", "pid": 9779, "process": {"exists": false, "pid": 9779}, "restart_count": 23, "restart_count_per_hour": 5, "rtsp_timeout_count": 32, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 24}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 102.236, "sample_count": 3018, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 132587, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T13:53:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "rtsp_timeout_count_per_hour>=10"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.022, "last_frame_pushed_at": "2026-07-02T13:53:12+09:00", "last_frame_read_age_sec": 0.098, "last_frame_read_at": "2026-07-02T13:53:11+09:00", "mjpeg": {"age_sec": 0.006, "exists": true, "mtime": "2026-07-02T13:53:12.036582+09:00", "mtime_epoch": 1782967992.036582, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 245155}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T13:53:12+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1095.418, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 115, "pushed_frames": 115, "read_frames": 114, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.84, "pushed": 3.84, "read": 3.807}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.325, "count": 115, "max_ms": 8.583, "p95_ms": 6.328}, "gstreamer_push": {"avg_ms": 1.329, "count": 115, "max_ms": 2.97, "p95_ms": 1.442}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.256, "count": 115, "max_ms": 76.213, "p95_ms": 67.871}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.418, "count": 115, "max_ms": 112.155, "p95_ms": 90.087}, "yolo11_640_fire_11class": {"avg_ms": 57.258, "count": 115, "max_ms": 80.951, "p95_ms": 75.116}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.975, "count": 115, "max_ms": 86.841, "p95_ms": 79.287}}, "truncated_count": 0}, "processing_total": {"avg_ms": 77.146, "count": 115, "max_ms": 125.905, "p95_ms": 103.919}, "read_interval": {"avg_ms": 262.569, "count": 114, "max_ms": 10169.434, "p95_ms": 127.424}, "triton_wait": {"avg_ms": 63.063, "count": 115, "max_ms": 112.167, "p95_ms": 90.1}, "visualize": {"avg_ms": 3.701, "count": 115, "max_ms": 4.74, "p95_ms": 4.085}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.948, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 32, "tracking_summary": 1104}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5312.367, "limit_mb": 8796093022207.996}, "detail": {"pid": 10038, "restart_count": 24, "start_count": 25}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 24, "h264_decode_error_count": 24, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T13:53:12+09:00", "last_error_summary": "[h264 @ 0x5629e1ee6340] error while decoding MB 76 31, bytestream -37 [in#0/rtsp @ 0x5629e1d1ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x5629e1d1ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T13:53:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T13:53:14+09:00", "pid": 10038, "process": {"exists": true, "fd_count": 5, "hwm_mb": 206.738, "name": "ffmpeg", "pid": 10038, "rss_mb": 206.738, "state": "S (sleeping)", "threads": 106}, "restart_count": 24, "restart_count_per_hour": 6, "rtsp_timeout_count": 32, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 25}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 102.217, "sample_count": 2974, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 132587, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T13:53:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 4.686, "last_frame_pushed_at": "2026-07-02T13:53:12+09:00", "last_frame_read_age_sec": 4.762, "last_frame_read_at": "2026-07-02T13:53:11+09:00", "mjpeg": {"age_sec": 4.67, "exists": true, "mtime": "2026-07-02T13:53:12.036582+09:00", "mtime_epoch": 1782967992.036582, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 245155}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T13:53:16+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1095.418, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 65, "pushed_frames": 65, "read_frames": 64, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.168, "pushed": 2.168, "read": 2.135}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.306, "count": 65, "max_ms": 8.583, "p95_ms": 6.272}, "gstreamer_push": {"avg_ms": 1.339, "count": 65, "max_ms": 2.97, "p95_ms": 1.447}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.795, "count": 65, "max_ms": 76.213, "p95_ms": 70.939}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.32, "count": 65, "max_ms": 108.764, "p95_ms": 93.957}, "yolo11_640_fire_11class": {"avg_ms": 57.779, "count": 65, "max_ms": 78.665, "p95_ms": 76.231}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.152, "count": 65, "max_ms": 86.841, "p95_ms": 82.534}}, "truncated_count": 0}, "processing_total": {"avg_ms": 78.216, "count": 65, "max_ms": 122.876, "p95_ms": 108.105}, "read_interval": {"avg_ms": 395.044, "count": 64, "max_ms": 10169.434, "p95_ms": 133.085}, "triton_wait": {"avg_ms": 64.108, "count": 65, "max_ms": 108.778, "p95_ms": 93.969}, "visualize": {"avg_ms": 3.703, "count": 65, "max_ms": 4.74, "p95_ms": 4.079}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.981, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 1137}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5180.27, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T13:41:14+09:00", "last_error_summary": "[h264 @ 0x55d875948540] error while decoding MB 13 37, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 29, "h264_decode_error_count": 47, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T13:41:14+09:00", "last_error_summary": "[h264 @ 0x55d875948540] error while decoding MB 13 37, bytestream -5", "last_exit_at": "2026-07-02T13:57:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 987.631, "last_started_at": "2026-07-02T13:41:17+09:00", "pid": 9931, "process": {"exists": false, "pid": 9931}, "restart_count": 37, "restart_count_per_hour": 4, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 38}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 136.713, "sample_count": 2771, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 136444, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T13:57:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-02T13:57:45+09:00", "last_frame_read_age_sec": 0.067, "last_frame_read_at": "2026-07-02T13:57:45+09:00", "mjpeg": {"age_sec": 0.066, "exists": true, "mtime": "2026-07-02T13:57:45.286559+09:00", "mtime_epoch": 1782968265.2865593, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200818}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T13:57:45+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1348.969, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 238, "pushed_frames": 238, "read_frames": 237, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.958, "pushed": 7.958, "read": 7.925}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.821, "count": 238, "max_ms": 10.374, "p95_ms": 7.378}, "gstreamer_push": {"avg_ms": 1.147, "count": 238, "max_ms": 1.582, "p95_ms": 1.357}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.589, "count": 238, "max_ms": 78.181, "p95_ms": 70.046}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.291, "count": 238, "max_ms": 105.008, "p95_ms": 89.093}, "yolo11_640_fire_11class": {"avg_ms": 56.954, "count": 238, "max_ms": 81.03, "p95_ms": 75.401}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.195, "count": 238, "max_ms": 93.727, "p95_ms": 84.156}}, "truncated_count": 0}, "processing_total": {"avg_ms": 112.464, "count": 238, "max_ms": 156.257, "p95_ms": 143.071}, "read_interval": {"avg_ms": 126.462, "count": 237, "max_ms": 169.672, "p95_ms": 156.601}, "triton_wait": {"avg_ms": 65.161, "count": 238, "max_ms": 105.022, "p95_ms": 90.739}, "visualize": {"avg_ms": 4.989, "count": 238, "max_ms": 9.18, "p95_ms": 5.807}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.906, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 1137}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5182.816, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 29, "h264_decode_error_count": 47, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T13:41:14+09:00", "last_error_summary": "[h264 @ 0x55d875948540] error while decoding MB 13 37, bytestream -5", "last_exit_at": "2026-07-02T13:57:45+09:00", "last_returncode": 0, "last_stable_duration_sec": 987.64, "last_started_at": "2026-07-02T13:41:17+09:00", "pid": 9931, "process": {"exists": false, "pid": 9931}, "restart_count": 37, "restart_count_per_hour": 4, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 38}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 136.713, "sample_count": 2771, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 136444, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T13:57:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.02, "last_frame_pushed_at": "2026-07-02T13:57:45+09:00", "last_frame_read_age_sec": 0.076, "last_frame_read_at": "2026-07-02T13:57:45+09:00", "mjpeg": {"age_sec": 0.074, "exists": true, "mtime": "2026-07-02T13:57:45.286559+09:00", "mtime_epoch": 1782968265.2865593, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200818}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T13:57:45+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1348.969, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 238, "pushed_frames": 238, "read_frames": 237, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.956, "pushed": 7.956, "read": 7.923}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.821, "count": 238, "max_ms": 10.374, "p95_ms": 7.378}, "gstreamer_push": {"avg_ms": 1.147, "count": 238, "max_ms": 1.582, "p95_ms": 1.357}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.589, "count": 238, "max_ms": 78.181, "p95_ms": 70.046}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.291, "count": 238, "max_ms": 105.008, "p95_ms": 89.093}, "yolo11_640_fire_11class": {"avg_ms": 56.954, "count": 238, "max_ms": 81.03, "p95_ms": 75.401}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.195, "count": 238, "max_ms": 93.727, "p95_ms": 84.156}}, "truncated_count": 0}, "processing_total": {"avg_ms": 112.464, "count": 238, "max_ms": 156.257, "p95_ms": 143.071}, "read_interval": {"avg_ms": 126.462, "count": 237, "max_ms": 169.672, "p95_ms": 156.601}, "triton_wait": {"avg_ms": 65.161, "count": 238, "max_ms": 105.022, "p95_ms": 90.739}, "visualize": {"avg_ms": 4.989, "count": 238, "max_ms": 9.18, "p95_ms": 5.807}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.914, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 1137}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5289.027, "limit_mb": 8796093022207.996}, "detail": {"pid": 10190, "restart_count": 38, "start_count": 39}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 29, "h264_decode_error_count": 47, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T13:41:14+09:00", "last_error_summary": "[h264 @ 0x55d875948540] error while decoding MB 13 37, bytestream -5", "last_exit_at": "2026-07-02T13:57:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T13:57:47+09:00", "pid": 10190, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.414, "name": "ffmpeg", "pid": 10190, "rss_mb": 139.414, "state": "S (sleeping)", "threads": 106}, "restart_count": 38, "restart_count_per_hour": 5, "rtsp_timeout_count": 12, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 39}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 136.833, "sample_count": 2722, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 136444, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T13:57:45+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.657, "last_frame_pushed_at": "2026-07-02T13:57:45+09:00", "last_frame_read_age_sec": 4.712, "last_frame_read_at": "2026-07-02T13:57:45+09:00", "mjpeg": {"age_sec": 4.639, "exists": true, "mtime": "2026-07-02T13:57:45.358561+09:00", "mtime_epoch": 1782968265.358561, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200450}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T13:57:50+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1348.969, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 203, "pushed_frames": 204, "read_frames": 203, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.767, "pushed": 6.8, "read": 6.767}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.751, "count": 203, "max_ms": 10.374, "p95_ms": 7.013}, "gstreamer_push": {"avg_ms": 1.153, "count": 204, "max_ms": 1.582, "p95_ms": 1.36}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.099, "count": 203, "max_ms": 78.181, "p95_ms": 67.954}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.198, "count": 203, "max_ms": 102.082, "p95_ms": 86.412}, "yolo11_640_fire_11class": {"avg_ms": 56.579, "count": 203, "max_ms": 81.03, "p95_ms": 73.048}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.763, "count": 203, "max_ms": 93.727, "p95_ms": 82.697}}, "truncated_count": 0}, "processing_total": {"avg_ms": 111.03, "count": 203, "max_ms": 156.257, "p95_ms": 143.071}, "read_interval": {"avg_ms": 125.11, "count": 203, "max_ms": 169.672, "p95_ms": 156.601}, "triton_wait": {"avg_ms": 64.825, "count": 203, "max_ms": 102.095, "p95_ms": 90.739}, "visualize": {"avg_ms": 4.932, "count": 203, "max_ms": 8.197, "p95_ms": 5.705}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 32, "tracking_summary": 1140}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5127.711, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55faeea1e000] error while decoding MB 119 25, bytestream -5 [in#0/rtsp @ 0x55faee9d6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55faee9d6e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 25, "h264_decode_error_count": 25, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T14:01:09+09:00", "last_error_summary": "[h264 @ 0x55faeea1e000] error while decoding MB 119 25, bytestream -5 [in#0/rtsp @ 0x55faee9d6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55faee9d6e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T13:53:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 475.036, "last_started_at": "2026-07-02T13:53:14+09:00", "pid": 10038, "process": {"exists": false, "pid": 10038}, "restart_count": 24, "restart_count_per_hour": 4, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 25}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.606, "sample_count": 2549, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 136868, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T14:01:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.078, "last_frame_pushed_at": "2026-07-02T14:01:09+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-02T14:01:09+09:00", "mjpeg": {"age_sec": 0.059, "exists": true, "mtime": "2026-07-02T14:01:09.682799+09:00", "mtime_epoch": 1782968469.682799, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 218728}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T14:01:09+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1097.223, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 10.863, "pushed": 10.863, "read": 11.768}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.215, "count": 12, "max_ms": 6.471, "p95_ms": 6.471}, "gstreamer_push": {"avg_ms": 1.284, "count": 12, "max_ms": 1.447, "p95_ms": 1.447}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.275, "count": 12, "max_ms": 72.805, "p95_ms": 72.805}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.953, "count": 12, "max_ms": 99.441, "p95_ms": 99.441}, "yolo11_640_fire_11class": {"avg_ms": 48.873, "count": 12, "max_ms": 70.292, "p95_ms": 70.292}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.567, "count": 12, "max_ms": 84.826, "p95_ms": 84.826}}, "truncated_count": 0}, "processing_total": {"avg_ms": 72.479, "count": 12, "max_ms": 113.781, "p95_ms": 113.781}, "read_interval": {"avg_ms": 2320.161, "count": 13, "max_ms": 29122.638, "p95_ms": 29122.638}, "triton_wait": {"avg_ms": 58.63, "count": 12, "max_ms": 99.454, "p95_ms": 99.454}, "visualize": {"avg_ms": 3.587, "count": 12, "max_ms": 4.351, "p95_ms": 4.351}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 1.105, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 32, "tracking_summary": 1140}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5125.273, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T14:01:09+09:00", "last_error_summary": "[h264 @ 0x55faeea1e000] error while decoding MB 119 25, bytestream -5 [in#0/rtsp @ 0x55faee9d6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55faee9d6e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 25, "h264_decode_error_count": 25, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T14:01:09+09:00", "last_error_summary": "[h264 @ 0x55faeea1e000] error while decoding MB 119 25, bytestream -5 [in#0/rtsp @ 0x55faee9d6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55faee9d6e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T14:01:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 475.059, "last_started_at": "2026-07-02T13:53:14+09:00", "pid": 10038, "process": {"exists": false, "pid": 10038}, "restart_count": 24, "restart_count_per_hour": 4, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 25}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.606, "sample_count": 2550, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 136869, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T14:01:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-02T14:01:09+09:00", "last_frame_read_age_sec": 0.088, "last_frame_read_at": "2026-07-02T14:01:09+09:00", "mjpeg": {"age_sec": 0.082, "exists": true, "mtime": "2026-07-02T14:01:09.682799+09:00", "mtime_epoch": 1782968469.682799, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 218728}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T14:01:09+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1097.223, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 11.53, "pushed": 11.53, "read": 11.53}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.281, "count": 13, "max_ms": 6.471, "p95_ms": 6.471}, "gstreamer_push": {"avg_ms": 1.3, "count": 13, "max_ms": 1.484, "p95_ms": 1.484}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.21, "count": 13, "max_ms": 72.805, "p95_ms": 72.805}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.757, "count": 13, "max_ms": 99.441, "p95_ms": 99.441}, "yolo11_640_fire_11class": {"avg_ms": 48.984, "count": 13, "max_ms": 70.292, "p95_ms": 70.292}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.73, "count": 13, "max_ms": 84.826, "p95_ms": 84.826}}, "truncated_count": 0}, "processing_total": {"avg_ms": 72.67, "count": 13, "max_ms": 113.781, "p95_ms": 113.781}, "read_interval": {"avg_ms": 2320.161, "count": 13, "max_ms": 29122.638, "p95_ms": 29122.638}, "triton_wait": {"avg_ms": 58.383, "count": 13, "max_ms": 99.454, "p95_ms": 99.454}, "visualize": {"avg_ms": 3.74, "count": 13, "max_ms": 5.581, "p95_ms": 5.581}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 1.128, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 32, "tracking_summary": 1140}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5127.754, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 25, "h264_decode_error_count": 25, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T14:01:09+09:00", "last_error_summary": "[h264 @ 0x55faeea1e000] error while decoding MB 119 25, bytestream -5 [in#0/rtsp @ 0x55faee9d6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55faee9d6e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T14:01:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 475.067, "last_started_at": "2026-07-02T13:53:14+09:00", "pid": 10038, "process": {"exists": false, "pid": 10038}, "restart_count": 24, "restart_count_per_hour": 4, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 25}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.606, "sample_count": 2550, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 136869, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T14:01:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.018, "last_frame_pushed_at": "2026-07-02T14:01:09+09:00", "last_frame_read_age_sec": 0.096, "last_frame_read_at": "2026-07-02T14:01:09+09:00", "mjpeg": {"age_sec": 0.09, "exists": true, "mtime": "2026-07-02T14:01:09.682799+09:00", "mtime_epoch": 1782968469.682799, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 218728}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T14:01:09+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1097.223, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 11.451, "pushed": 11.451, "read": 11.451}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.281, "count": 13, "max_ms": 6.471, "p95_ms": 6.471}, "gstreamer_push": {"avg_ms": 1.3, "count": 13, "max_ms": 1.484, "p95_ms": 1.484}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.21, "count": 13, "max_ms": 72.805, "p95_ms": 72.805}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.757, "count": 13, "max_ms": 99.441, "p95_ms": 99.441}, "yolo11_640_fire_11class": {"avg_ms": 48.984, "count": 13, "max_ms": 70.292, "p95_ms": 70.292}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.73, "count": 13, "max_ms": 84.826, "p95_ms": 84.826}}, "truncated_count": 0}, "processing_total": {"avg_ms": 72.67, "count": 13, "max_ms": 113.781, "p95_ms": 113.781}, "read_interval": {"avg_ms": 2320.161, "count": 13, "max_ms": 29122.638, "p95_ms": 29122.638}, "triton_wait": {"avg_ms": 58.383, "count": 13, "max_ms": 99.454, "p95_ms": 99.454}, "visualize": {"avg_ms": 3.74, "count": 13, "max_ms": 5.581, "p95_ms": 5.581}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 1.135, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 32, "tracking_summary": 1140}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5307.754, "limit_mb": 8796093022207.996}, "detail": {"pid": 10297, "restart_count": 25, "start_count": 26}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 25, "h264_decode_error_count": 25, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T14:01:09+09:00", "last_error_summary": "[h264 @ 0x55faeea1e000] error while decoding MB 119 25, bytestream -5 [in#0/rtsp @ 0x55faee9d6e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55faee9d6e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T14:01:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T14:01:11+09:00", "pid": 10297, "process": {"exists": true, "fd_count": 5, "hwm_mb": 210.27, "name": "ffmpeg", "pid": 10297, "rss_mb": 210.27, "state": "S (sleeping)", "threads": 106}, "restart_count": 25, "restart_count_per_hour": 5, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 26}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 143.779, "sample_count": 2507, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 136869, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T14:01:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 4.094, "last_frame_pushed_at": "2026-07-02T14:01:09+09:00", "last_frame_read_age_sec": 4.171, "last_frame_read_at": "2026-07-02T14:01:09+09:00", "mjpeg": {"age_sec": 4.073, "exists": true, "mtime": "2026-07-02T14:01:09.774801+09:00", "mtime_epoch": 1782968469.774801, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 213834}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T14:01:13+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1097.223, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.495, "pushed": 2.495, "read": 2.495}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.281, "count": 13, "max_ms": 6.471, "p95_ms": 6.471}, "gstreamer_push": {"avg_ms": 1.3, "count": 13, "max_ms": 1.484, "p95_ms": 1.484}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.21, "count": 13, "max_ms": 72.805, "p95_ms": 72.805}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.757, "count": 13, "max_ms": 99.441, "p95_ms": 99.441}, "yolo11_640_fire_11class": {"avg_ms": 48.984, "count": 13, "max_ms": 70.292, "p95_ms": 70.292}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.73, "count": 13, "max_ms": 84.826, "p95_ms": 84.826}}, "truncated_count": 0}, "processing_total": {"avg_ms": 72.67, "count": 13, "max_ms": 113.781, "p95_ms": 113.781}, "read_interval": {"avg_ms": 2320.161, "count": 13, "max_ms": 29122.638, "p95_ms": 29122.638}, "triton_wait": {"avg_ms": 58.383, "count": 13, "max_ms": 99.454, "p95_ms": 99.454}, "visualize": {"avg_ms": 3.74, "count": 13, "max_ms": 5.581, "p95_ms": 5.581}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 5.211, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 1194}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5193.918, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 7, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x562160fe8600] left block unavailable for requested intra mode [h264 @ 0x562160fe8600] error while decoding MB 0 25, bytestream 6851 [h264 @ 0x562161c6b440] cabac decode of qscale diff failed at 41 31 [h264 @ 0x562161c6b440] error while decoding MB 41 31, bytestream 8299 [h264 @ 0x562160fd4500] error while decoding MB 11 6, bytestream -18 [h264 @ 0x5621610db8c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5621610db8c0] error while decoding MB 0 39, bytestream 3161 [h", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 30, "h264_decode_error_count": 54, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T14:10:01+09:00", "last_error_summary": "[h264 @ 0x562160fe8600] left block unavailable for requested intra mode [h264 @ 0x562160fe8600] error while decoding MB 0 25, bytestream 6851 [h264 @ 0x562161c6b440] cabac decode of qscale diff failed at 41 31 [h264 @ 0x562161c6b440] error while decoding MB 41 31, bytestream 8299 [h264 @ 0x562160fd4500] error while decoding MB 11 6, bytestream -18 [h264 @ 0x5621610db8c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5621610db8c0] error while decoding MB 0 39, bytestream 3161 [h", "last_exit_at": "2026-07-02T13:57:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 733.191, "last_started_at": "2026-07-02T13:57:47+09:00", "pid": 10190, "process": {"exists": false, "pid": 10190}, "restart_count": 38, "restart_count_per_hour": 3, "rtsp_timeout_count": 14, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 39}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 133.832, "sample_count": 2621, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 143343, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T14:10:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.067, "last_frame_pushed_at": "2026-07-02T14:10:01+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-02T14:10:01+09:00", "mjpeg": {"age_sec": 0.05, "exists": true, "mtime": "2026-07-02T14:10:01.137329+09:00", "mtime_epoch": 1782969001.1373293, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197999}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T14:10:01+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1352.062, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 111, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.701, "pushed": 3.701, "read": 3.701}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.824, "count": 111, "max_ms": 5.378, "p95_ms": 4.418}, "gstreamer_push": {"avg_ms": 1.319, "count": 111, "max_ms": 2.088, "p95_ms": 1.453}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.145, "count": 111, "max_ms": 69.418, "p95_ms": 57.758}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.732, "count": 111, "max_ms": 84.269, "p95_ms": 79.783}, "yolo11_640_fire_11class": {"avg_ms": 49.334, "count": 111, "max_ms": 70.813, "p95_ms": 58.462}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.111, "count": 111, "max_ms": 82.135, "p95_ms": 66.737}}, "truncated_count": 0}, "processing_total": {"avg_ms": 71.023, "count": 111, "max_ms": 96.965, "p95_ms": 93.167}, "read_interval": {"avg_ms": 270.322, "count": 111, "max_ms": 10338.246, "p95_ms": 111.31}, "triton_wait": {"avg_ms": 58.472, "count": 111, "max_ms": 84.282, "p95_ms": 80.265}, "visualize": {"avg_ms": 3.709, "count": 111, "max_ms": 4.992, "p95_ms": 4.139}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.989, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 1194}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5192.941, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T14:10:01+09:00", "last_error_summary": "[h264 @ 0x562160fe8600] left block unavailable for requested intra mode [h264 @ 0x562160fe8600] error while decoding MB 0 25, bytestream 6851 [h264 @ 0x562161c6b440] cabac decode of qscale diff failed at 41 31 [h264 @ 0x562161c6b440] error while decoding MB 41 31, bytestream 8299 [h264 @ 0x562160fd4500] error while decoding MB 11 6, bytestream -18 [h264 @ 0x5621610db8c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5621610db8c0] error while decoding MB 0 39, bytestream 3161 [h", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 30, "h264_decode_error_count": 54, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T14:10:01+09:00", "last_error_summary": "[h264 @ 0x562160fe8600] left block unavailable for requested intra mode [h264 @ 0x562160fe8600] error while decoding MB 0 25, bytestream 6851 [h264 @ 0x562161c6b440] cabac decode of qscale diff failed at 41 31 [h264 @ 0x562161c6b440] error while decoding MB 41 31, bytestream 8299 [h264 @ 0x562160fd4500] error while decoding MB 11 6, bytestream -18 [h264 @ 0x5621610db8c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5621610db8c0] error while decoding MB 0 39, bytestream 3161 [h", "last_exit_at": "2026-07-02T14:10:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 733.225, "last_started_at": "2026-07-02T13:57:47+09:00", "pid": 10190, "process": {"exists": false, "pid": 10190}, "restart_count": 38, "restart_count_per_hour": 3, "rtsp_timeout_count": 14, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 39}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 133.832, "sample_count": 2621, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 143344, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T14:10:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T14:10:01+09:00", "last_frame_read_age_sec": 0.089, "last_frame_read_at": "2026-07-02T14:10:01+09:00", "mjpeg": {"age_sec": 0.084, "exists": true, "mtime": "2026-07-02T14:10:01.137329+09:00", "mtime_epoch": 1782969001.1373293, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197999}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T14:10:01+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1352.062, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 110, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.707, "pushed": 3.707, "read": 3.674}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.862, "count": 111, "max_ms": 7.921, "p95_ms": 4.431}, "gstreamer_push": {"avg_ms": 1.319, "count": 111, "max_ms": 2.088, "p95_ms": 1.453}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.241, "count": 111, "max_ms": 69.418, "p95_ms": 60.192}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.752, "count": 111, "max_ms": 84.269, "p95_ms": 79.783}, "yolo11_640_fire_11class": {"avg_ms": 49.343, "count": 111, "max_ms": 70.813, "p95_ms": 58.462}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.191, "count": 111, "max_ms": 82.135, "p95_ms": 66.737}}, "truncated_count": 0}, "processing_total": {"avg_ms": 71.101, "count": 111, "max_ms": 96.965, "p95_ms": 93.167}, "read_interval": {"avg_ms": 271.976, "count": 110, "max_ms": 10338.246, "p95_ms": 111.31}, "triton_wait": {"avg_ms": 58.505, "count": 111, "max_ms": 84.282, "p95_ms": 80.265}, "visualize": {"avg_ms": 3.717, "count": 111, "max_ms": 4.992, "p95_ms": 4.406}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.942, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 1194}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5192.934, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 30, "h264_decode_error_count": 54, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T14:10:01+09:00", "last_error_summary": "[h264 @ 0x562160fe8600] left block unavailable for requested intra mode [h264 @ 0x562160fe8600] error while decoding MB 0 25, bytestream 6851 [h264 @ 0x562161c6b440] cabac decode of qscale diff failed at 41 31 [h264 @ 0x562161c6b440] error while decoding MB 41 31, bytestream 8299 [h264 @ 0x562160fd4500] error while decoding MB 11 6, bytestream -18 [h264 @ 0x5621610db8c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5621610db8c0] error while decoding MB 0 39, bytestream 3161 [h", "last_exit_at": "2026-07-02T14:10:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 733.233, "last_started_at": "2026-07-02T13:57:47+09:00", "pid": 10190, "process": {"exists": false, "pid": 10190}, "restart_count": 38, "restart_count_per_hour": 3, "rtsp_timeout_count": 14, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 39}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 133.832, "sample_count": 2621, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 143344, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T14:10:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-02T14:10:01+09:00", "last_frame_read_age_sec": 0.098, "last_frame_read_at": "2026-07-02T14:10:01+09:00", "mjpeg": {"age_sec": 0.093, "exists": true, "mtime": "2026-07-02T14:10:01.137329+09:00", "mtime_epoch": 1782969001.1373293, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197999}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T14:10:01+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1352.062, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 110, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.706, "pushed": 3.706, "read": 3.673}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.862, "count": 111, "max_ms": 7.921, "p95_ms": 4.431}, "gstreamer_push": {"avg_ms": 1.319, "count": 111, "max_ms": 2.088, "p95_ms": 1.453}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.241, "count": 111, "max_ms": 69.418, "p95_ms": 60.192}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.752, "count": 111, "max_ms": 84.269, "p95_ms": 79.783}, "yolo11_640_fire_11class": {"avg_ms": 49.343, "count": 111, "max_ms": 70.813, "p95_ms": 58.462}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.191, "count": 111, "max_ms": 82.135, "p95_ms": 66.737}}, "truncated_count": 0}, "processing_total": {"avg_ms": 71.101, "count": 111, "max_ms": 96.965, "p95_ms": 93.167}, "read_interval": {"avg_ms": 271.976, "count": 110, "max_ms": 10338.246, "p95_ms": 111.31}, "triton_wait": {"avg_ms": 58.505, "count": 111, "max_ms": 84.282, "p95_ms": 80.265}, "visualize": {"avg_ms": 3.717, "count": 111, "max_ms": 4.992, "p95_ms": 4.406}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.95, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 1194}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5303.492, "limit_mb": 8796093022207.996}, "detail": {"pid": 10449, "restart_count": 39, "start_count": 40}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 30, "h264_decode_error_count": 54, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T14:10:01+09:00", "last_error_summary": "[h264 @ 0x562160fe8600] left block unavailable for requested intra mode [h264 @ 0x562160fe8600] error while decoding MB 0 25, bytestream 6851 [h264 @ 0x562161c6b440] cabac decode of qscale diff failed at 41 31 [h264 @ 0x562161c6b440] error while decoding MB 41 31, bytestream 8299 [h264 @ 0x562160fd4500] error while decoding MB 11 6, bytestream -18 [h264 @ 0x5621610db8c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x5621610db8c0] error while decoding MB 0 39, bytestream 3161 [h", "last_exit_at": "2026-07-02T14:10:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T14:10:04+09:00", "pid": 10449, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.281, "name": "ffmpeg", "pid": 10449, "rss_mb": 139.281, "state": "S (sleeping)", "threads": 106}, "restart_count": 39, "restart_count_per_hour": 4, "rtsp_timeout_count": 14, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 40}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 133.878, "sample_count": 2572, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 143344, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T14:10:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.988, "last_frame_pushed_at": "2026-07-02T14:10:01+09:00", "last_frame_read_age_sec": 5.071, "last_frame_read_at": "2026-07-02T14:10:01+09:00", "mjpeg": {"age_sec": 4.97, "exists": true, "mtime": "2026-07-02T14:10:01.233331+09:00", "mtime_epoch": 1782969001.2333312, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199017}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T14:10:06+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1352.062, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 55, "pushed_frames": 55, "read_frames": 54, "select_timeouts": 20, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.836, "pushed": 1.836, "read": 1.803}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.875, "count": 55, "max_ms": 7.921, "p95_ms": 4.431}, "gstreamer_push": {"avg_ms": 1.295, "count": 55, "max_ms": 1.458, "p95_ms": 1.428}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.415, "count": 55, "max_ms": 62.923, "p95_ms": 57.758}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 57.502, "count": 55, "max_ms": 84.269, "p95_ms": 80.253}, "yolo11_640_fire_11class": {"avg_ms": 49.415, "count": 55, "max_ms": 67.555, "p95_ms": 57.738}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.635, "count": 55, "max_ms": 71.983, "p95_ms": 66.737}}, "truncated_count": 0}, "processing_total": {"avg_ms": 70.542, "count": 55, "max_ms": 96.965, "p95_ms": 93.167}, "read_interval": {"avg_ms": 462.176, "count": 54, "max_ms": 10338.246, "p95_ms": 112.989}, "triton_wait": {"avg_ms": 57.987, "count": 55, "max_ms": 84.282, "p95_ms": 80.265}, "visualize": {"avg_ms": 3.714, "count": 55, "max_ms": 4.689, "p95_ms": 4.139}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.953, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 1221}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5205.469, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 8, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x5579c906a940] cabac decode of qscale diff failed at 15 39 [h264 @ 0x5579c906a940] error while decoding MB 15 39, bytestream 5190 [h264 @ 0x5579c84eca80] cabac decode of qscale diff failed at 18 39 [h264 @ 0x5579c84eca80] error while decoding MB 18 39, bytestream 3475 [h264 @ 0x5579c8f9bac0] cabac decode of qscale diff failed at 26 43 [h264 @ 0x5579c8f9bac0] error while decoding MB 26 43, bytestream 1782 [h264 @ 0x5579c906a940] error while decoding MB 32 44, bytestream -20 [h264 @ 0x557", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 31, "h264_decode_error_count": 62, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T14:15:55+09:00", "last_error_summary": "[h264 @ 0x5579c906a940] cabac decode of qscale diff failed at 15 39 [h264 @ 0x5579c906a940] error while decoding MB 15 39, bytestream 5190 [h264 @ 0x5579c84eca80] cabac decode of qscale diff failed at 18 39 [h264 @ 0x5579c84eca80] error while decoding MB 18 39, bytestream 3475 [h264 @ 0x5579c8f9bac0] cabac decode of qscale diff failed at 26 43 [h264 @ 0x5579c8f9bac0] error while decoding MB 26 43, bytestream 1782 [h264 @ 0x5579c906a940] error while decoding MB 32 44, bytestream -20 [h264 @ 0x557", "last_exit_at": "2026-07-02T14:10:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 351.566, "last_started_at": "2026-07-02T14:10:04+09:00", "pid": 10449, "process": {"exists": false, "pid": 10449}, "restart_count": 39, "restart_count_per_hour": 4, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 40}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 134.391, "sample_count": 2706, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 146532, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T14:15:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.071, "last_frame_pushed_at": "2026-07-02T14:15:55+09:00", "last_frame_read_age_sec": 0.059, "last_frame_read_at": "2026-07-02T14:15:55+09:00", "mjpeg": {"age_sec": 0.053, "exists": true, "mtime": "2026-07-02T14:15:55.715214+09:00", "mtime_epoch": 1782969355.7152145, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200029}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T14:15:55+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1353.867, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 110, "pushed_frames": 110, "read_frames": 111, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.668, "pushed": 3.668, "read": 3.701}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.854, "count": 110, "max_ms": 5.169, "p95_ms": 4.66}, "gstreamer_push": {"avg_ms": 1.325, "count": 110, "max_ms": 1.572, "p95_ms": 1.473}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.916, "count": 110, "max_ms": 72.257, "p95_ms": 68.896}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.144, "count": 110, "max_ms": 99.658, "p95_ms": 88.28}, "yolo11_640_fire_11class": {"avg_ms": 53.968, "count": 110, "max_ms": 74.287, "p95_ms": 71.603}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.251, "count": 110, "max_ms": 84.267, "p95_ms": 79.574}}, "truncated_count": 0}, "processing_total": {"avg_ms": 76.351, "count": 110, "max_ms": 112.192, "p95_ms": 101.076}, "read_interval": {"avg_ms": 270.376, "count": 111, "max_ms": 10125.16, "p95_ms": 121.871}, "triton_wait": {"avg_ms": 63.664, "count": 110, "max_ms": 99.685, "p95_ms": 88.289}, "visualize": {"avg_ms": 3.765, "count": 110, "max_ms": 5.234, "p95_ms": 4.403}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 1221}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5205.18, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T14:15:55+09:00", "last_error_summary": "[h264 @ 0x5579c906a940] cabac decode of qscale diff failed at 15 39 [h264 @ 0x5579c906a940] error while decoding MB 15 39, bytestream 5190 [h264 @ 0x5579c84eca80] cabac decode of qscale diff failed at 18 39 [h264 @ 0x5579c84eca80] error while decoding MB 18 39, bytestream 3475 [h264 @ 0x5579c8f9bac0] cabac decode of qscale diff failed at 26 43 [h264 @ 0x5579c8f9bac0] error while decoding MB 26 43, bytestream 1782 [h264 @ 0x5579c906a940] error while decoding MB 32 44, bytestream -20 [h264 @ 0x557", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 31, "h264_decode_error_count": 62, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T14:15:55+09:00", "last_error_summary": "[h264 @ 0x5579c906a940] cabac decode of qscale diff failed at 15 39 [h264 @ 0x5579c906a940] error while decoding MB 15 39, bytestream 5190 [h264 @ 0x5579c84eca80] cabac decode of qscale diff failed at 18 39 [h264 @ 0x5579c84eca80] error while decoding MB 18 39, bytestream 3475 [h264 @ 0x5579c8f9bac0] cabac decode of qscale diff failed at 26 43 [h264 @ 0x5579c8f9bac0] error while decoding MB 26 43, bytestream 1782 [h264 @ 0x5579c906a940] error while decoding MB 32 44, bytestream -20 [h264 @ 0x557", "last_exit_at": "2026-07-02T14:15:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 351.592, "last_started_at": "2026-07-02T14:10:04+09:00", "pid": 10449, "process": {"exists": false, "pid": 10449}, "restart_count": 39, "restart_count_per_hour": 4, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 40}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 134.391, "sample_count": 2707, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 146533, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T14:15:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T14:15:55+09:00", "last_frame_read_age_sec": 0.084, "last_frame_read_at": "2026-07-02T14:15:55+09:00", "mjpeg": {"age_sec": 0.078, "exists": true, "mtime": "2026-07-02T14:15:55.715214+09:00", "mtime_epoch": 1782969355.7152145, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200029}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T14:15:55+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1353.867, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 110, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.706, "pushed": 3.706, "read": 3.673}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.867, "count": 111, "max_ms": 5.354, "p95_ms": 4.797}, "gstreamer_push": {"avg_ms": 1.325, "count": 111, "max_ms": 1.572, "p95_ms": 1.473}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.852, "count": 111, "max_ms": 72.257, "p95_ms": 68.896}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.064, "count": 111, "max_ms": 99.658, "p95_ms": 88.28}, "yolo11_640_fire_11class": {"avg_ms": 53.892, "count": 111, "max_ms": 74.287, "p95_ms": 71.603}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.161, "count": 111, "max_ms": 84.267, "p95_ms": 79.574}}, "truncated_count": 0}, "processing_total": {"avg_ms": 76.339, "count": 111, "max_ms": 112.192, "p95_ms": 101.076}, "read_interval": {"avg_ms": 272.105, "count": 110, "max_ms": 10125.16, "p95_ms": 121.871}, "triton_wait": {"avg_ms": 63.571, "count": 111, "max_ms": 99.685, "p95_ms": 88.289}, "visualize": {"avg_ms": 3.818, "count": 111, "max_ms": 9.637, "p95_ms": 4.568}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.95, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 1221}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5205.023, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 31, "h264_decode_error_count": 62, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T14:15:55+09:00", "last_error_summary": "[h264 @ 0x5579c906a940] cabac decode of qscale diff failed at 15 39 [h264 @ 0x5579c906a940] error while decoding MB 15 39, bytestream 5190 [h264 @ 0x5579c84eca80] cabac decode of qscale diff failed at 18 39 [h264 @ 0x5579c84eca80] error while decoding MB 18 39, bytestream 3475 [h264 @ 0x5579c8f9bac0] cabac decode of qscale diff failed at 26 43 [h264 @ 0x5579c8f9bac0] error while decoding MB 26 43, bytestream 1782 [h264 @ 0x5579c906a940] error while decoding MB 32 44, bytestream -20 [h264 @ 0x557", "last_exit_at": "2026-07-02T14:15:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 351.6, "last_started_at": "2026-07-02T14:10:04+09:00", "pid": 10449, "process": {"exists": false, "pid": 10449}, "restart_count": 39, "restart_count_per_hour": 4, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 40}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 134.391, "sample_count": 2707, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 146533, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T14:15:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-02T14:15:55+09:00", "last_frame_read_age_sec": 0.092, "last_frame_read_at": "2026-07-02T14:15:55+09:00", "mjpeg": {"age_sec": 0.086, "exists": true, "mtime": "2026-07-02T14:15:55.715214+09:00", "mtime_epoch": 1782969355.7152145, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200029}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T14:15:55+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1353.867, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 110, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.705, "pushed": 3.705, "read": 3.672}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.867, "count": 111, "max_ms": 5.354, "p95_ms": 4.797}, "gstreamer_push": {"avg_ms": 1.325, "count": 111, "max_ms": 1.572, "p95_ms": 1.473}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.852, "count": 111, "max_ms": 72.257, "p95_ms": 68.896}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.064, "count": 111, "max_ms": 99.658, "p95_ms": 88.28}, "yolo11_640_fire_11class": {"avg_ms": 53.892, "count": 111, "max_ms": 74.287, "p95_ms": 71.603}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.161, "count": 111, "max_ms": 84.267, "p95_ms": 79.574}}, "truncated_count": 0}, "processing_total": {"avg_ms": 76.339, "count": 111, "max_ms": 112.192, "p95_ms": 101.076}, "read_interval": {"avg_ms": 272.105, "count": 110, "max_ms": 10125.16, "p95_ms": 121.871}, "triton_wait": {"avg_ms": 63.571, "count": 111, "max_ms": 99.685, "p95_ms": 88.289}, "visualize": {"avg_ms": 3.818, "count": 111, "max_ms": 9.637, "p95_ms": 4.568}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.959, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 1221}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5310.934, "limit_mb": 8796093022207.996}, "detail": {"pid": 10556, "restart_count": 40, "start_count": 41}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 31, "h264_decode_error_count": 62, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T14:15:55+09:00", "last_error_summary": "[h264 @ 0x5579c906a940] cabac decode of qscale diff failed at 15 39 [h264 @ 0x5579c906a940] error while decoding MB 15 39, bytestream 5190 [h264 @ 0x5579c84eca80] cabac decode of qscale diff failed at 18 39 [h264 @ 0x5579c84eca80] error while decoding MB 18 39, bytestream 3475 [h264 @ 0x5579c8f9bac0] cabac decode of qscale diff failed at 26 43 [h264 @ 0x5579c8f9bac0] error while decoding MB 26 43, bytestream 1782 [h264 @ 0x5579c906a940] error while decoding MB 32 44, bytestream -20 [h264 @ 0x557", "last_exit_at": "2026-07-02T14:15:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T14:15:57+09:00", "pid": 10556, "process": {"exists": true, "fd_count": 5, "hwm_mb": 137.238, "name": "ffmpeg", "pid": 10556, "rss_mb": 137.238, "state": "S (sleeping)", "threads": 106}, "restart_count": 40, "restart_count_per_hour": 5, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 41}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 133.608, "sample_count": 2676, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 146533, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T14:15:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.162, "last_frame_pushed_at": "2026-07-02T14:15:55+09:00", "last_frame_read_age_sec": 4.239, "last_frame_read_at": "2026-07-02T14:15:55+09:00", "mjpeg": {"age_sec": 4.142, "exists": true, "mtime": "2026-07-02T14:15:55.807215+09:00", "mtime_epoch": 1782969355.807215, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 192983}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T14:15:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1353.867, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 65, "pushed_frames": 65, "read_frames": 64, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.171, "pushed": 2.171, "read": 2.137}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.897, "count": 65, "max_ms": 5.354, "p95_ms": 4.809}, "gstreamer_push": {"avg_ms": 1.328, "count": 65, "max_ms": 1.572, "p95_ms": 1.465}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.523, "count": 65, "max_ms": 72.257, "p95_ms": 70.664}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.924, "count": 65, "max_ms": 99.658, "p95_ms": 89.03}, "yolo11_640_fire_11class": {"avg_ms": 54.902, "count": 65, "max_ms": 74.134, "p95_ms": 72.04}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.572, "count": 65, "max_ms": 84.267, "p95_ms": 80.248}}, "truncated_count": 0}, "processing_total": {"avg_ms": 78.434, "count": 65, "max_ms": 112.192, "p95_ms": 101.649}, "read_interval": {"avg_ms": 402.662, "count": 64, "max_ms": 10125.16, "p95_ms": 124.098}, "triton_wait": {"avg_ms": 65.51, "count": 65, "max_ms": 99.685, "p95_ms": 89.043}, "visualize": {"avg_ms": 3.877, "count": 65, "max_ms": 9.637, "p95_ms": 4.568}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.946, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 32, "tracking_summary": 1288}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5162.605, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[NULL @ 0x564b43b84200] missing picture in access unit with size 18 [h264 @ 0x564b43d37480] no frame! [in#0/rtsp @ 0x564b43b7fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x564b43b7fe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x564b43b84840] Decoding error: Invalid data found when processing input", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 26, "h264_decode_error_count": 26, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T14:31:53+09:00", "last_error_summary": "[NULL @ 0x564b43b84200] missing picture in access unit with size 18 [h264 @ 0x564b43d37480] no frame! [in#0/rtsp @ 0x564b43b7fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x564b43b7fe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x564b43b84840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T14:01:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 1841.884, "last_started_at": "2026-07-02T14:01:11+09:00", "pid": 10297, "process": {"exists": false, "pid": 10297}, "restart_count": 25, "restart_count_per_hour": 3, "rtsp_timeout_count": 36, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 26}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 142.111, "sample_count": 2430, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 154663, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T14:31:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.077, "last_frame_pushed_at": "2026-07-02T14:31:53+09:00", "last_frame_read_age_sec": 0.069, "last_frame_read_at": "2026-07-02T14:31:53+09:00", "mjpeg": {"age_sec": 0.057, "exists": true, "mtime": "2026-07-02T14:31:53.673438+09:00", "mtime_epoch": 1782970313.6734383, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 253944}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T14:31:53+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1105.473, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 44, "pushed_frames": 44, "read_frames": 44, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.476, "pushed": 1.476, "read": 1.476}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 10.949, "count": 44, "max_ms": 13.53, "p95_ms": 12.755}, "gstreamer_push": {"avg_ms": 1.003, "count": 44, "max_ms": 1.811, "p95_ms": 1.109}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 61.269, "count": 44, "max_ms": 91.887, "p95_ms": 85.322}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.332, "count": 44, "max_ms": 93.245, "p95_ms": 89.618}, "yolo11_640_fire_11class": {"avg_ms": 58.726, "count": 44, "max_ms": 90.568, "p95_ms": 78.964}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.104, "count": 44, "max_ms": 101.507, "p95_ms": 96.513}}, "truncated_count": 0}, "processing_total": {"avg_ms": 163.456, "count": 44, "max_ms": 205.414, "p95_ms": 204.953}, "read_interval": {"avg_ms": 680.469, "count": 44, "max_ms": 12450.836, "p95_ms": 217.231}, "triton_wait": {"avg_ms": 70.889, "count": 44, "max_ms": 101.52, "p95_ms": 96.524}, "visualize": {"avg_ms": 5.376, "count": 44, "max_ms": 6.803, "p95_ms": 6.272}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.807, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 32, "tracking_summary": 1288}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5166.93, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T14:31:53+09:00", "last_error_summary": "[NULL @ 0x564b43b84200] missing picture in access unit with size 18 [h264 @ 0x564b43d37480] no frame! [in#0/rtsp @ 0x564b43b7fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x564b43b7fe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x564b43b84840] Decoding error: Invalid data found when processing input", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 26, "h264_decode_error_count": 26, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T14:31:53+09:00", "last_error_summary": "[NULL @ 0x564b43b84200] missing picture in access unit with size 18 [h264 @ 0x564b43d37480] no frame! [in#0/rtsp @ 0x564b43b7fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x564b43b7fe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x564b43b84840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T14:31:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 1842.022, "last_started_at": "2026-07-02T14:01:11+09:00", "pid": 10297, "process": {"exists": false, "pid": 10297}, "restart_count": 25, "restart_count_per_hour": 3, "rtsp_timeout_count": 36, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 26}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 142.173, "sample_count": 2429, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 154664, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T14:31:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T14:31:53+09:00", "last_frame_read_age_sec": 0.207, "last_frame_read_at": "2026-07-02T14:31:53+09:00", "mjpeg": {"age_sec": 0.195, "exists": true, "mtime": "2026-07-02T14:31:53.673438+09:00", "mtime_epoch": 1782970313.6734383, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 253944}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T14:31:53+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1105.473, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 45, "pushed_frames": 45, "read_frames": 44, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.503, "pushed": 1.503, "read": 1.469}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 11.055, "count": 44, "max_ms": 16.465, "p95_ms": 12.926}, "gstreamer_push": {"avg_ms": 1.001, "count": 45, "max_ms": 1.811, "p95_ms": 1.109}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.81, "count": 44, "max_ms": 91.887, "p95_ms": 85.322}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.672, "count": 44, "max_ms": 93.245, "p95_ms": 89.618}, "yolo11_640_fire_11class": {"avg_ms": 58.329, "count": 44, "max_ms": 90.568, "p95_ms": 78.964}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.437, "count": 44, "max_ms": 101.507, "p95_ms": 96.513}}, "truncated_count": 0}, "processing_total": {"avg_ms": 164.327, "count": 45, "max_ms": 205.414, "p95_ms": 204.953}, "read_interval": {"avg_ms": 680.469, "count": 44, "max_ms": 12450.836, "p95_ms": 217.231}, "triton_wait": {"avg_ms": 70.223, "count": 44, "max_ms": 101.52, "p95_ms": 96.524}, "visualize": {"avg_ms": 5.423, "count": 44, "max_ms": 8.189, "p95_ms": 6.432}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.943, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 32, "tracking_summary": 1288}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5166.922, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 26, "h264_decode_error_count": 26, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T14:31:53+09:00", "last_error_summary": "[NULL @ 0x564b43b84200] missing picture in access unit with size 18 [h264 @ 0x564b43d37480] no frame! [in#0/rtsp @ 0x564b43b7fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x564b43b7fe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x564b43b84840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T14:31:53+09:00", "last_returncode": 0, "last_stable_duration_sec": 1842.026, "last_started_at": "2026-07-02T14:01:11+09:00", "pid": 10297, "process": {"exists": false, "pid": 10297}, "restart_count": 25, "restart_count_per_hour": 3, "rtsp_timeout_count": 36, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 26}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 142.173, "sample_count": 2429, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 154664, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T14:31:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T14:31:53+09:00", "last_frame_read_age_sec": 0.211, "last_frame_read_at": "2026-07-02T14:31:53+09:00", "mjpeg": {"age_sec": 0.199, "exists": true, "mtime": "2026-07-02T14:31:53.673438+09:00", "mtime_epoch": 1782970313.6734383, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 253944}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T14:31:53+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1105.473, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 45, "pushed_frames": 45, "read_frames": 44, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.503, "pushed": 1.503, "read": 1.469}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 11.055, "count": 44, "max_ms": 16.465, "p95_ms": 12.926}, "gstreamer_push": {"avg_ms": 1.001, "count": 45, "max_ms": 1.811, "p95_ms": 1.109}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.81, "count": 44, "max_ms": 91.887, "p95_ms": 85.322}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.672, "count": 44, "max_ms": 93.245, "p95_ms": 89.618}, "yolo11_640_fire_11class": {"avg_ms": 58.329, "count": 44, "max_ms": 90.568, "p95_ms": 78.964}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 67.437, "count": 44, "max_ms": 101.507, "p95_ms": 96.513}}, "truncated_count": 0}, "processing_total": {"avg_ms": 164.327, "count": 45, "max_ms": 205.414, "p95_ms": 204.953}, "read_interval": {"avg_ms": 680.469, "count": 44, "max_ms": 12450.836, "p95_ms": 217.231}, "triton_wait": {"avg_ms": 70.223, "count": 44, "max_ms": 101.52, "p95_ms": 96.524}, "visualize": {"avg_ms": 5.423, "count": 44, "max_ms": 8.189, "p95_ms": 6.432}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.946, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 32, "tracking_summary": 1288}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5189.938, "limit_mb": 8796093022207.996}, "detail": {"pid": 10663, "restart_count": 26, "start_count": 27}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 26, "h264_decode_error_count": 26, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T14:31:53+09:00", "last_error_summary": "[NULL @ 0x564b43b84200] missing picture in access unit with size 18 [h264 @ 0x564b43d37480] no frame! [in#0/rtsp @ 0x564b43b7fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x564b43b7fe00] Error retrieving a packet from demuxer: Connection timed out [vist#0:0/h264 @ 0x564b43b84840] Decoding error: Invalid data found when processing input", "last_exit_at": "2026-07-02T14:31:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T14:31:56+09:00", "pid": 10663, "process": {"exists": true, "fd_count": 5, "hwm_mb": 64.777, "name": "ffmpeg", "pid": 10663, "rss_mb": 64.777, "state": "S (sleeping)", "threads": 19}, "restart_count": 26, "restart_count_per_hour": 4, "rtsp_timeout_count": 36, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 27}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 142.301, "sample_count": 2386, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 154664, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T14:31:53+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 4.309, "last_frame_pushed_at": "2026-07-02T14:31:53+09:00", "last_frame_read_age_sec": 4.513, "last_frame_read_at": "2026-07-02T14:31:53+09:00", "mjpeg": {"age_sec": 4.293, "exists": true, "mtime": "2026-07-02T14:31:53.881441+09:00", "mtime_epoch": 1782970313.881441, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 253676}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T14:31:58+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1105.473, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 21, "pushed_frames": 21, "read_frames": 20, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.701, "pushed": 0.701, "read": 0.667}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 10.789, "count": 20, "max_ms": 16.465, "p95_ms": 12.277}, "gstreamer_push": {"avg_ms": 1.045, "count": 21, "max_ms": 1.811, "p95_ms": 1.124}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 60.781, "count": 20, "max_ms": 91.887, "p95_ms": 74.805}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.566, "count": 20, "max_ms": 93.245, "p95_ms": 90.801}, "yolo11_640_fire_11class": {"avg_ms": 59.284, "count": 20, "max_ms": 90.568, "p95_ms": 73.274}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 68.508, "count": 20, "max_ms": 101.507, "p95_ms": 92.679}}, "truncated_count": 0}, "processing_total": {"avg_ms": 160.051, "count": 21, "max_ms": 202.67, "p95_ms": 197.852}, "read_interval": {"avg_ms": 1281.728, "count": 20, "max_ms": 12450.836, "p95_ms": 10157.743}, "triton_wait": {"avg_ms": 71.495, "count": 20, "max_ms": 101.52, "p95_ms": 93.258}, "visualize": {"avg_ms": 5.175, "count": 20, "max_ms": 8.189, "p95_ms": 6.166}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.977, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 1370}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5196.918, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5583de8a2840] error while decoding MB 63 34, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 27, "h264_decode_error_count": 27, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T14:51:58+09:00", "last_error_summary": "[h264 @ 0x5583de8a2840] error while decoding MB 63 34, bytestream -7", "last_exit_at": "2026-07-02T14:31:56+09:00", "last_returncode": 0, "last_stable_duration_sec": 1202.069, "last_started_at": "2026-07-02T14:31:56+09:00", "pid": 10663, "process": {"exists": false, "pid": 10663}, "restart_count": 26, "restart_count_per_hour": 3, "rtsp_timeout_count": 36, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 27}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 166.096, "sample_count": 2579, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 164466, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T14:51:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.079, "last_frame_pushed_at": "2026-07-02T14:51:58+09:00", "last_frame_read_age_sec": 0.063, "last_frame_read_at": "2026-07-02T14:51:58+09:00", "mjpeg": {"age_sec": 0.061, "exists": true, "mtime": "2026-07-02T14:51:58.181723+09:00", "mtime_epoch": 1782971518.1817234, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 231503}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T14:51:58+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1110.629, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 339, "pushed_frames": 339, "read_frames": 339, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 11.315, "pushed": 11.315, "read": 11.315}, "hints": ["triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.839, "count": 339, "max_ms": 8.948, "p95_ms": 6.782}, "gstreamer_push": {"avg_ms": 1.295, "count": 339, "max_ms": 2.28, "p95_ms": 1.453}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.767, "count": 339, "max_ms": 74.843, "p95_ms": 65.508}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.066, "count": 339, "max_ms": 103.374, "p95_ms": 80.594}, "yolo11_640_fire_11class": {"avg_ms": 53.991, "count": 339, "max_ms": 79.775, "p95_ms": 69.455}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.861, "count": 339, "max_ms": 83.661, "p95_ms": 76.559}}, "truncated_count": 0}, "processing_total": {"avg_ms": 71.853, "count": 339, "max_ms": 117.747, "p95_ms": 96.501}, "read_interval": {"avg_ms": 88.504, "count": 339, "max_ms": 137.269, "p95_ms": 114.974}, "triton_wait": {"avg_ms": 57.371, "count": 339, "max_ms": 103.388, "p95_ms": 81.567}, "visualize": {"avg_ms": 3.675, "count": 339, "max_ms": 6.012, "p95_ms": 4.082}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.961, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 1370}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5196.305, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T14:51:58+09:00", "last_error_summary": "[h264 @ 0x5583de8a2840] error while decoding MB 63 34, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 27, "h264_decode_error_count": 27, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T14:51:58+09:00", "last_error_summary": "[h264 @ 0x5583de8a2840] error while decoding MB 63 34, bytestream -7", "last_exit_at": "2026-07-02T14:51:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 1202.11, "last_started_at": "2026-07-02T14:31:56+09:00", "pid": 10663, "process": {"exists": false, "pid": 10663}, "restart_count": 26, "restart_count_per_hour": 3, "rtsp_timeout_count": 36, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 27}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 166.096, "sample_count": 2580, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 164467, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T14:51:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-02T14:51:58+09:00", "last_frame_read_age_sec": 0.103, "last_frame_read_at": "2026-07-02T14:51:58+09:00", "mjpeg": {"age_sec": 0.101, "exists": true, "mtime": "2026-07-02T14:51:58.181723+09:00", "mtime_epoch": 1782971518.1817234, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 231503}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T14:51:58+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1110.629, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 339, "pushed_frames": 340, "read_frames": 339, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 11.3, "pushed": 11.334, "read": 11.3}, "hints": ["triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.839, "count": 339, "max_ms": 8.948, "p95_ms": 6.782}, "gstreamer_push": {"avg_ms": 1.295, "count": 340, "max_ms": 2.28, "p95_ms": 1.451}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.794, "count": 339, "max_ms": 74.843, "p95_ms": 65.988}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.028, "count": 339, "max_ms": 103.374, "p95_ms": 80.522}, "yolo11_640_fire_11class": {"avg_ms": 53.984, "count": 339, "max_ms": 79.775, "p95_ms": 69.356}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.852, "count": 339, "max_ms": 83.661, "p95_ms": 76.384}}, "truncated_count": 0}, "processing_total": {"avg_ms": 71.816, "count": 339, "max_ms": 117.747, "p95_ms": 96.244}, "read_interval": {"avg_ms": 88.504, "count": 339, "max_ms": 137.269, "p95_ms": 114.974}, "triton_wait": {"avg_ms": 57.334, "count": 339, "max_ms": 103.388, "p95_ms": 80.753}, "visualize": {"avg_ms": 3.675, "count": 339, "max_ms": 6.012, "p95_ms": 4.082}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 1370}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5196.371, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 27, "h264_decode_error_count": 27, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T14:51:58+09:00", "last_error_summary": "[h264 @ 0x5583de8a2840] error while decoding MB 63 34, bytestream -7", "last_exit_at": "2026-07-02T14:51:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 1202.121, "last_started_at": "2026-07-02T14:31:56+09:00", "pid": 10663, "process": {"exists": false, "pid": 10663}, "restart_count": 26, "restart_count_per_hour": 3, "rtsp_timeout_count": 36, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 27}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 166.096, "sample_count": 2579, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 164467, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T14:51:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.02, "last_frame_pushed_at": "2026-07-02T14:51:58+09:00", "last_frame_read_age_sec": 0.115, "last_frame_read_at": "2026-07-02T14:51:58+09:00", "mjpeg": {"age_sec": 0.113, "exists": true, "mtime": "2026-07-02T14:51:58.181723+09:00", "mtime_epoch": 1782971518.1817234, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 231503}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T14:51:58+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1110.629, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 339, "pushed_frames": 339, "read_frames": 339, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 11.303, "pushed": 11.303, "read": 11.303}, "hints": ["triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.839, "count": 339, "max_ms": 8.948, "p95_ms": 6.782}, "gstreamer_push": {"avg_ms": 1.295, "count": 339, "max_ms": 2.28, "p95_ms": 1.453}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.794, "count": 339, "max_ms": 74.843, "p95_ms": 65.988}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.028, "count": 339, "max_ms": 103.374, "p95_ms": 80.522}, "yolo11_640_fire_11class": {"avg_ms": 53.984, "count": 339, "max_ms": 79.775, "p95_ms": 69.356}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 52.852, "count": 339, "max_ms": 83.661, "p95_ms": 76.384}}, "truncated_count": 0}, "processing_total": {"avg_ms": 71.816, "count": 339, "max_ms": 117.747, "p95_ms": 96.244}, "read_interval": {"avg_ms": 88.504, "count": 339, "max_ms": 137.269, "p95_ms": 114.974}, "triton_wait": {"avg_ms": 57.334, "count": 339, "max_ms": 103.388, "p95_ms": 80.753}, "visualize": {"avg_ms": 3.675, "count": 339, "max_ms": 6.012, "p95_ms": 4.082}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 1370}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5211.434, "limit_mb": 8796093022207.996}, "detail": {"pid": 10875, "restart_count": 27, "start_count": 28}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 27, "h264_decode_error_count": 27, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T14:51:58+09:00", "last_error_summary": "[h264 @ 0x5583de8a2840] error while decoding MB 63 34, bytestream -7", "last_exit_at": "2026-07-02T14:52:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T14:52:00+09:00", "pid": 10875, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.75, "name": "ffmpeg", "pid": 10875, "rss_mb": 57.75, "state": "S (sleeping)", "threads": 1}, "restart_count": 27, "restart_count_per_hour": 4, "rtsp_timeout_count": 36, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 28}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 166.193, "sample_count": 2530, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 164467, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T14:51:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 4.702, "last_frame_pushed_at": "2026-07-02T14:51:58+09:00", "last_frame_read_age_sec": 4.797, "last_frame_read_at": "2026-07-02T14:51:58+09:00", "mjpeg": {"age_sec": 4.683, "exists": true, "mtime": "2026-07-02T14:51:58.293726+09:00", "mtime_epoch": 1782971518.293726, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 228004}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T14:52:02+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1110.629, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 283, "pushed_frames": 283, "read_frames": 282, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 9.447, "pushed": 9.447, "read": 9.413}, "hints": ["triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.856, "count": 283, "max_ms": 8.948, "p95_ms": 6.896}, "gstreamer_push": {"avg_ms": 1.298, "count": 283, "max_ms": 2.28, "p95_ms": 1.46}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.304, "count": 283, "max_ms": 74.843, "p95_ms": 66.155}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 54.72, "count": 283, "max_ms": 103.374, "p95_ms": 80.742}, "yolo11_640_fire_11class": {"avg_ms": 54.622, "count": 283, "max_ms": 79.775, "p95_ms": 69.489}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.639, "count": 283, "max_ms": 83.661, "p95_ms": 76.751}}, "truncated_count": 0}, "processing_total": {"avg_ms": 72.81, "count": 283, "max_ms": 117.747, "p95_ms": 96.603}, "read_interval": {"avg_ms": 89.455, "count": 282, "max_ms": 137.269, "p95_ms": 115.403}, "triton_wait": {"avg_ms": 58.277, "count": 283, "max_ms": 103.388, "p95_ms": 81.938}, "visualize": {"avg_ms": 3.685, "count": 283, "max_ms": 6.012, "p95_ms": 4.116}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.957, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 1406}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5237.668, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T14:15:55+09:00", "last_error_summary": "[h264 @ 0x5579c906a940] cabac decode of qscale diff failed at 15 39 [h264 @ 0x5579c906a940] error while decoding MB 15 39, bytestream 5190 [h264 @ 0x5579c84eca80] cabac decode of qscale diff failed at 18 39 [h264 @ 0x5579c84eca80] error while decoding MB 18 39, bytestream 3475 [h264 @ 0x5579c8f9bac0] cabac decode of qscale diff failed at 26 43 [h264 @ 0x5579c8f9bac0] error while decoding MB 26 43, bytestream 1782 [h264 @ 0x5579c906a940] error while decoding MB 32 44, bytestream -20 [h264 @ 0x557", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 31, "h264_decode_error_count": 62, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T14:15:55+09:00", "last_error_summary": "[h264 @ 0x5579c906a940] cabac decode of qscale diff failed at 15 39 [h264 @ 0x5579c906a940] error while decoding MB 15 39, bytestream 5190 [h264 @ 0x5579c84eca80] cabac decode of qscale diff failed at 18 39 [h264 @ 0x5579c84eca80] error while decoding MB 18 39, bytestream 3475 [h264 @ 0x5579c8f9bac0] cabac decode of qscale diff failed at 26 43 [h264 @ 0x5579c8f9bac0] error while decoding MB 26 43, bytestream 1782 [h264 @ 0x5579c906a940] error while decoding MB 32 44, bytestream -20 [h264 @ 0x557", "last_exit_at": "2026-07-02T14:54:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 2284.27, "last_started_at": "2026-07-02T14:15:57+09:00", "pid": 10556, "process": {"exists": false, "pid": 10556}, "restart_count": 40, "restart_count_per_hour": 3, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 41}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.813, "sample_count": 2421, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 168781, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T14:54:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-02T14:54:02+09:00", "last_frame_read_age_sec": 0.102, "last_frame_read_at": "2026-07-02T14:54:02+09:00", "mjpeg": {"age_sec": 0.101, "exists": true, "mtime": "2026-07-02T14:54:02.116596+09:00", "mtime_epoch": 1782971642.1165957, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196245}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T14:54:02+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1364.438, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 83, "pushed_frames": 83, "read_frames": 82, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.767, "pushed": 2.767, "read": 2.734}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.856, "count": 83, "max_ms": 5.543, "p95_ms": 4.416}, "gstreamer_push": {"avg_ms": 1.338, "count": 83, "max_ms": 2.07, "p95_ms": 1.491}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.094, "count": 83, "max_ms": 79.652, "p95_ms": 72.244}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.002, "count": 83, "max_ms": 95.474, "p95_ms": 85.926}, "yolo11_640_fire_11class": {"avg_ms": 54.147, "count": 83, "max_ms": 82.628, "p95_ms": 73.112}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.028, "count": 83, "max_ms": 89.951, "p95_ms": 80.425}}, "truncated_count": 0}, "processing_total": {"avg_ms": 76.211, "count": 83, "max_ms": 108.164, "p95_ms": 99.922}, "read_interval": {"avg_ms": 365.302, "count": 82, "max_ms": 9704.908, "p95_ms": 181.549}, "triton_wait": {"avg_ms": 63.532, "count": 83, "max_ms": 95.487, "p95_ms": 85.939}, "visualize": {"avg_ms": 3.713, "count": 83, "max_ms": 4.731, "p95_ms": 4.231}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.993, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 1406}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5235.602, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 31, "h264_decode_error_count": 62, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T14:15:55+09:00", "last_error_summary": "[h264 @ 0x5579c906a940] cabac decode of qscale diff failed at 15 39 [h264 @ 0x5579c906a940] error while decoding MB 15 39, bytestream 5190 [h264 @ 0x5579c84eca80] cabac decode of qscale diff failed at 18 39 [h264 @ 0x5579c84eca80] error while decoding MB 18 39, bytestream 3475 [h264 @ 0x5579c8f9bac0] cabac decode of qscale diff failed at 26 43 [h264 @ 0x5579c8f9bac0] error while decoding MB 26 43, bytestream 1782 [h264 @ 0x5579c906a940] error while decoding MB 32 44, bytestream -20 [h264 @ 0x557", "last_exit_at": "2026-07-02T14:54:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 2284.281, "last_started_at": "2026-07-02T14:15:57+09:00", "pid": 10556, "process": {"exists": false, "pid": 10556}, "restart_count": 40, "restart_count_per_hour": 3, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 41}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.813, "sample_count": 2421, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 168781, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T14:54:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.02, "last_frame_pushed_at": "2026-07-02T14:54:02+09:00", "last_frame_read_age_sec": 0.113, "last_frame_read_at": "2026-07-02T14:54:02+09:00", "mjpeg": {"age_sec": 0.112, "exists": true, "mtime": "2026-07-02T14:54:02.116596+09:00", "mtime_epoch": 1782971642.1165957, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196245}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T14:54:02+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1364.438, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 82, "pushed_frames": 82, "read_frames": 82, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.734, "pushed": 2.734, "read": 2.734}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.859, "count": 82, "max_ms": 5.543, "p95_ms": 4.416}, "gstreamer_push": {"avg_ms": 1.338, "count": 82, "max_ms": 2.07, "p95_ms": 1.491}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.178, "count": 82, "max_ms": 79.652, "p95_ms": 72.244}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.13, "count": 82, "max_ms": 95.474, "p95_ms": 85.926}, "yolo11_640_fire_11class": {"avg_ms": 54.243, "count": 82, "max_ms": 82.628, "p95_ms": 73.112}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.142, "count": 82, "max_ms": 89.951, "p95_ms": 80.425}}, "truncated_count": 0}, "processing_total": {"avg_ms": 76.365, "count": 82, "max_ms": 108.164, "p95_ms": 99.922}, "read_interval": {"avg_ms": 365.302, "count": 82, "max_ms": 9704.908, "p95_ms": 181.549}, "triton_wait": {"avg_ms": 63.68, "count": 82, "max_ms": 95.487, "p95_ms": 85.939}, "visualize": {"avg_ms": 3.715, "count": 82, "max_ms": 4.731, "p95_ms": 4.231}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.989, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 1406}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5339.027, "limit_mb": 8796093022207.996}, "detail": {"pid": 11027, "restart_count": 41, "start_count": 42}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 31, "h264_decode_error_count": 62, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T14:15:55+09:00", "last_error_summary": "[h264 @ 0x5579c906a940] cabac decode of qscale diff failed at 15 39 [h264 @ 0x5579c906a940] error while decoding MB 15 39, bytestream 5190 [h264 @ 0x5579c84eca80] cabac decode of qscale diff failed at 18 39 [h264 @ 0x5579c84eca80] error while decoding MB 18 39, bytestream 3475 [h264 @ 0x5579c8f9bac0] cabac decode of qscale diff failed at 26 43 [h264 @ 0x5579c8f9bac0] error while decoding MB 26 43, bytestream 1782 [h264 @ 0x5579c906a940] error while decoding MB 32 44, bytestream -20 [h264 @ 0x557", "last_exit_at": "2026-07-02T14:54:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T14:54:04+09:00", "pid": 11027, "process": {"exists": true, "fd_count": 5, "hwm_mb": 137.488, "name": "ffmpeg", "pid": 11027, "rss_mb": 137.488, "state": "S (sleeping)", "threads": 106}, "restart_count": 41, "restart_count_per_hour": 4, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 42}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 99.835, "sample_count": 2381, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 168781, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T14:54:02+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.061, "last_frame_pushed_at": "2026-07-02T14:54:02+09:00", "last_frame_read_age_sec": 4.155, "last_frame_read_at": "2026-07-02T14:54:02+09:00", "mjpeg": {"age_sec": 4.038, "exists": true, "mtime": "2026-07-02T14:54:02.232598+09:00", "mtime_epoch": 1782971642.2325983, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196627}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T14:54:06+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1364.438, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 39, "pushed_frames": 39, "read_frames": 39, "select_timeouts": 21, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.456, "pushed": 1.456, "read": 1.456}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.881, "count": 39, "max_ms": 5.499, "p95_ms": 4.714}, "gstreamer_push": {"avg_ms": 1.327, "count": 39, "max_ms": 1.445, "p95_ms": 1.437}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 58.664, "count": 39, "max_ms": 79.652, "p95_ms": 76.957}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 65.369, "count": 39, "max_ms": 95.474, "p95_ms": 93.817}, "yolo11_640_fire_11class": {"avg_ms": 59.378, "count": 39, "max_ms": 82.628, "p95_ms": 80.176}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.794, "count": 39, "max_ms": 89.951, "p95_ms": 85.271}}, "truncated_count": 0}, "processing_total": {"avg_ms": 81.131, "count": 39, "max_ms": 108.164, "p95_ms": 106.441}, "read_interval": {"avg_ms": 670.936, "count": 39, "max_ms": 9704.908, "p95_ms": 9162.922}, "triton_wait": {"avg_ms": 68.436, "count": 39, "max_ms": 95.487, "p95_ms": 93.83}, "visualize": {"avg_ms": 3.713, "count": 39, "max_ms": 4.244, "p95_ms": 4.231}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 26.781, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 1424}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5248.402, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T14:15:55+09:00", "last_error_summary": "[h264 @ 0x5579c906a940] cabac decode of qscale diff failed at 15 39 [h264 @ 0x5579c906a940] error while decoding MB 15 39, bytestream 5190 [h264 @ 0x5579c84eca80] cabac decode of qscale diff failed at 18 39 [h264 @ 0x5579c84eca80] error while decoding MB 18 39, bytestream 3475 [h264 @ 0x5579c8f9bac0] cabac decode of qscale diff failed at 26 43 [h264 @ 0x5579c8f9bac0] error while decoding MB 26 43, bytestream 1782 [h264 @ 0x5579c906a940] error while decoding MB 32 44, bytestream -20 [h264 @ 0x557", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 31, "h264_decode_error_count": 62, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T14:15:55+09:00", "last_error_summary": "[h264 @ 0x5579c906a940] cabac decode of qscale diff failed at 15 39 [h264 @ 0x5579c906a940] error while decoding MB 15 39, bytestream 5190 [h264 @ 0x5579c84eca80] cabac decode of qscale diff failed at 18 39 [h264 @ 0x5579c84eca80] error while decoding MB 18 39, bytestream 3475 [h264 @ 0x5579c8f9bac0] cabac decode of qscale diff failed at 26 43 [h264 @ 0x5579c8f9bac0] error while decoding MB 26 43, bytestream 1782 [h264 @ 0x5579c906a940] error while decoding MB 32 44, bytestream -20 [h264 @ 0x557", "last_exit_at": "2026-07-02T14:58:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 262.935, "last_started_at": "2026-07-02T14:54:04+09:00", "pid": 11027, "process": {"exists": false, "pid": 11027}, "restart_count": 41, "restart_count_per_hour": 3, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 42}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.823, "sample_count": 2244, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 170882, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T14:58:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-02T14:58:27+09:00", "last_frame_read_age_sec": 0.079, "last_frame_read_at": "2026-07-02T14:58:27+09:00", "mjpeg": {"age_sec": 0.075, "exists": true, "mtime": "2026-07-02T14:58:27.130263+09:00", "mtime_epoch": 1782971907.130263, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199497}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T14:58:27+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1365.211, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 216, "pushed_frames": 216, "read_frames": 215, "select_timeouts": 9, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.208, "pushed": 7.208, "read": 7.174}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.868, "count": 216, "max_ms": 6.013, "p95_ms": 4.554}, "gstreamer_push": {"avg_ms": 1.348, "count": 216, "max_ms": 2.456, "p95_ms": 1.508}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.919, "count": 216, "max_ms": 74.423, "p95_ms": 67.925}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.66, "count": 216, "max_ms": 98.926, "p95_ms": 84.442}, "yolo11_640_fire_11class": {"avg_ms": 51.77, "count": 216, "max_ms": 77.726, "p95_ms": 70.17}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.514, "count": 216, "max_ms": 84.12, "p95_ms": 78.081}}, "truncated_count": 0}, "processing_total": {"avg_ms": 73.56, "count": 216, "max_ms": 111.572, "p95_ms": 97.712}, "read_interval": {"avg_ms": 139.334, "count": 215, "max_ms": 3107.289, "p95_ms": 115.326}, "triton_wait": {"avg_ms": 60.852, "count": 216, "max_ms": 98.938, "p95_ms": 84.454}, "visualize": {"avg_ms": 3.768, "count": 216, "max_ms": 5.529, "p95_ms": 4.654}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.968, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 1424}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5248.301, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 31, "h264_decode_error_count": 62, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T14:15:55+09:00", "last_error_summary": "[h264 @ 0x5579c906a940] cabac decode of qscale diff failed at 15 39 [h264 @ 0x5579c906a940] error while decoding MB 15 39, bytestream 5190 [h264 @ 0x5579c84eca80] cabac decode of qscale diff failed at 18 39 [h264 @ 0x5579c84eca80] error while decoding MB 18 39, bytestream 3475 [h264 @ 0x5579c8f9bac0] cabac decode of qscale diff failed at 26 43 [h264 @ 0x5579c8f9bac0] error while decoding MB 26 43, bytestream 1782 [h264 @ 0x5579c906a940] error while decoding MB 32 44, bytestream -20 [h264 @ 0x557", "last_exit_at": "2026-07-02T14:58:27+09:00", "last_returncode": 0, "last_stable_duration_sec": 262.948, "last_started_at": "2026-07-02T14:54:04+09:00", "pid": 11027, "process": {"exists": false, "pid": 11027}, "restart_count": 41, "restart_count_per_hour": 3, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 42}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 107.823, "sample_count": 2244, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 170882, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T14:58:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.023, "last_frame_pushed_at": "2026-07-02T14:58:27+09:00", "last_frame_read_age_sec": 0.092, "last_frame_read_at": "2026-07-02T14:58:27+09:00", "mjpeg": {"age_sec": 0.007, "exists": true, "mtime": "2026-07-02T14:58:27.210265+09:00", "mtime_epoch": 1782971907.2102647, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199856}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T14:58:27+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1365.211, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 216, "pushed_frames": 216, "read_frames": 215, "select_timeouts": 9, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.205, "pushed": 7.205, "read": 7.171}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.868, "count": 216, "max_ms": 6.013, "p95_ms": 4.554}, "gstreamer_push": {"avg_ms": 1.348, "count": 216, "max_ms": 2.456, "p95_ms": 1.508}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.919, "count": 216, "max_ms": 74.423, "p95_ms": 67.925}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.66, "count": 216, "max_ms": 98.926, "p95_ms": 84.442}, "yolo11_640_fire_11class": {"avg_ms": 51.77, "count": 216, "max_ms": 77.726, "p95_ms": 70.17}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.514, "count": 216, "max_ms": 84.12, "p95_ms": 78.081}}, "truncated_count": 0}, "processing_total": {"avg_ms": 73.56, "count": 216, "max_ms": 111.572, "p95_ms": 97.712}, "read_interval": {"avg_ms": 139.334, "count": 215, "max_ms": 3107.289, "p95_ms": 115.326}, "triton_wait": {"avg_ms": 60.852, "count": 216, "max_ms": 98.938, "p95_ms": 84.454}, "visualize": {"avg_ms": 3.768, "count": 216, "max_ms": 5.529, "p95_ms": 4.654}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.981, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 1424}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5363.973, "limit_mb": 8796093022207.996}, "detail": {"pid": 11134, "restart_count": 42, "start_count": 43}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 31, "h264_decode_error_count": 62, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T14:15:55+09:00", "last_error_summary": "[h264 @ 0x5579c906a940] cabac decode of qscale diff failed at 15 39 [h264 @ 0x5579c906a940] error while decoding MB 15 39, bytestream 5190 [h264 @ 0x5579c84eca80] cabac decode of qscale diff failed at 18 39 [h264 @ 0x5579c84eca80] error while decoding MB 18 39, bytestream 3475 [h264 @ 0x5579c8f9bac0] cabac decode of qscale diff failed at 26 43 [h264 @ 0x5579c8f9bac0] error while decoding MB 26 43, bytestream 1782 [h264 @ 0x5579c906a940] error while decoding MB 32 44, bytestream -20 [h264 @ 0x557", "last_exit_at": "2026-07-02T14:58:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T14:58:30+09:00", "pid": 11134, "process": {"exists": true, "fd_count": 5, "hwm_mb": 146.492, "name": "ffmpeg", "pid": 11134, "rss_mb": 146.492, "state": "S (sleeping)", "threads": 106}, "restart_count": 42, "restart_count_per_hour": 4, "rtsp_timeout_count": 16, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 43}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 108.29, "sample_count": 2183, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 170882, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T14:58:27+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 5.032, "last_frame_pushed_at": "2026-07-02T14:58:27+09:00", "last_frame_read_age_sec": 5.101, "last_frame_read_at": "2026-07-02T14:58:27+09:00", "mjpeg": {"age_sec": 5.016, "exists": true, "mtime": "2026-07-02T14:58:27.210265+09:00", "mtime_epoch": 1782971907.2102647, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199856}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T14:58:32+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1365.211, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 185, "pushed_frames": 185, "read_frames": 185, "select_timeouts": 7, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.25, "pushed": 6.25, "read": 6.25}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.87, "count": 185, "max_ms": 6.013, "p95_ms": 4.554}, "gstreamer_push": {"avg_ms": 1.339, "count": 185, "max_ms": 2.456, "p95_ms": 1.463}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.204, "count": 185, "max_ms": 74.423, "p95_ms": 68.507}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.759, "count": 185, "max_ms": 98.926, "p95_ms": 84.442}, "yolo11_640_fire_11class": {"avg_ms": 52.184, "count": 185, "max_ms": 77.726, "p95_ms": 70.641}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.131, "count": 185, "max_ms": 84.12, "p95_ms": 78.66}}, "truncated_count": 0}, "processing_total": {"avg_ms": 73.914, "count": 185, "max_ms": 111.572, "p95_ms": 97.712}, "read_interval": {"avg_ms": 147.62, "count": 185, "max_ms": 3107.289, "p95_ms": 118.653}, "triton_wait": {"avg_ms": 61.232, "count": 185, "max_ms": 98.938, "p95_ms": 84.454}, "visualize": {"avg_ms": 3.764, "count": 185, "max_ms": 5.529, "p95_ms": 4.703}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.598, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 1435}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5245.875, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 5, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55be11e2b180] cabac decode of qscale diff failed at 24 40 [h264 @ 0x55be11e2b180] error while decoding MB 24 40, bytestream 5238 [h264 @ 0x55be129e5080] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55be129e5080] error while decoding MB 0 32, bytestream 1842 [h264 @ 0x55be12916200] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55be12916200] error while decoding MB 0 22, bytestream 5628 [h264 @ 0x55be12847440] left block unavailable for requested intr", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 32, "h264_decode_error_count": 67, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T15:01:44+09:00", "last_error_summary": "[h264 @ 0x55be11e2b180] cabac decode of qscale diff failed at 24 40 [h264 @ 0x55be11e2b180] error while decoding MB 24 40, bytestream 5238 [h264 @ 0x55be129e5080] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55be129e5080] error while decoding MB 0 32, bytestream 1842 [h264 @ 0x55be12916200] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55be12916200] error while decoding MB 0 22, bytestream 5628 [h264 @ 0x55be12847440] left block unavailable for requested intr", "last_exit_at": "2026-07-02T14:58:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 194.04, "last_started_at": "2026-07-02T14:58:30+09:00", "pid": 11134, "process": {"exists": false, "pid": 11134}, "restart_count": 42, "restart_count_per_hour": 4, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 43}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.69, "sample_count": 2227, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 172253, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T15:01:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.069, "last_frame_pushed_at": "2026-07-02T15:01:44+09:00", "last_frame_read_age_sec": 0.057, "last_frame_read_at": "2026-07-02T15:01:44+09:00", "mjpeg": {"age_sec": 0.051, "exists": true, "mtime": "2026-07-02T15:01:44.214190+09:00", "mtime_epoch": 1782972104.21419, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206989}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T15:01:44+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1365.984, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 115, "pushed_frames": 115, "read_frames": 115, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.843, "pushed": 3.843, "read": 3.843}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.894, "count": 115, "max_ms": 5.34, "p95_ms": 4.946}, "gstreamer_push": {"avg_ms": 1.308, "count": 115, "max_ms": 1.47, "p95_ms": 1.422}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.575, "count": 115, "max_ms": 79.721, "p95_ms": 68.108}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.79, "count": 115, "max_ms": 103.106, "p95_ms": 83.688}, "yolo11_640_fire_11class": {"avg_ms": 51.507, "count": 115, "max_ms": 84.737, "p95_ms": 70.373}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.919, "count": 115, "max_ms": 91.642, "p95_ms": 80.715}}, "truncated_count": 0}, "processing_total": {"avg_ms": 74.278, "count": 115, "max_ms": 116.064, "p95_ms": 97.498}, "read_interval": {"avg_ms": 260.468, "count": 115, "max_ms": 10119.104, "p95_ms": 114.828}, "triton_wait": {"avg_ms": 61.752, "count": 115, "max_ms": 103.119, "p95_ms": 84.817}, "visualize": {"avg_ms": 3.707, "count": 115, "max_ms": 4.785, "p95_ms": 4.339}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.925, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 1435}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5245.781, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T15:01:44+09:00", "last_error_summary": "[h264 @ 0x55be11e2b180] cabac decode of qscale diff failed at 24 40 [h264 @ 0x55be11e2b180] error while decoding MB 24 40, bytestream 5238 [h264 @ 0x55be129e5080] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55be129e5080] error while decoding MB 0 32, bytestream 1842 [h264 @ 0x55be12916200] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55be12916200] error while decoding MB 0 22, bytestream 5628 [h264 @ 0x55be12847440] left block unavailable for requested intr", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 32, "h264_decode_error_count": 67, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T15:01:44+09:00", "last_error_summary": "[h264 @ 0x55be11e2b180] cabac decode of qscale diff failed at 24 40 [h264 @ 0x55be11e2b180] error while decoding MB 24 40, bytestream 5238 [h264 @ 0x55be129e5080] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55be129e5080] error while decoding MB 0 32, bytestream 1842 [h264 @ 0x55be12916200] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55be12916200] error while decoding MB 0 22, bytestream 5628 [h264 @ 0x55be12847440] left block unavailable for requested intr", "last_exit_at": "2026-07-02T15:01:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 194.061, "last_started_at": "2026-07-02T14:58:30+09:00", "pid": 11134, "process": {"exists": false, "pid": 11134}, "restart_count": 42, "restart_count_per_hour": 4, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 43}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.69, "sample_count": 2227, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 172254, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T15:01:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T15:01:44+09:00", "last_frame_read_age_sec": 0.078, "last_frame_read_at": "2026-07-02T15:01:44+09:00", "mjpeg": {"age_sec": 0.072, "exists": true, "mtime": "2026-07-02T15:01:44.214190+09:00", "mtime_epoch": 1782972104.21419, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206989}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T15:01:44+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1365.984, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 115, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.874, "pushed": 3.874, "read": 3.84}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.894, "count": 116, "max_ms": 5.34, "p95_ms": 4.946}, "gstreamer_push": {"avg_ms": 1.308, "count": 116, "max_ms": 1.47, "p95_ms": 1.422}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.521, "count": 116, "max_ms": 79.721, "p95_ms": 68.108}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.722, "count": 116, "max_ms": 103.106, "p95_ms": 83.688}, "yolo11_640_fire_11class": {"avg_ms": 51.49, "count": 116, "max_ms": 84.737, "p95_ms": 70.373}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.831, "count": 116, "max_ms": 91.642, "p95_ms": 80.715}}, "truncated_count": 0}, "processing_total": {"avg_ms": 74.236, "count": 116, "max_ms": 116.064, "p95_ms": 97.498}, "read_interval": {"avg_ms": 260.468, "count": 115, "max_ms": 10119.104, "p95_ms": 114.828}, "triton_wait": {"avg_ms": 61.667, "count": 116, "max_ms": 103.119, "p95_ms": 84.817}, "visualize": {"avg_ms": 3.742, "count": 116, "max_ms": 7.711, "p95_ms": 4.344}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.945, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 1435}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5245.773, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 32, "h264_decode_error_count": 67, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T15:01:44+09:00", "last_error_summary": "[h264 @ 0x55be11e2b180] cabac decode of qscale diff failed at 24 40 [h264 @ 0x55be11e2b180] error while decoding MB 24 40, bytestream 5238 [h264 @ 0x55be129e5080] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55be129e5080] error while decoding MB 0 32, bytestream 1842 [h264 @ 0x55be12916200] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55be12916200] error while decoding MB 0 22, bytestream 5628 [h264 @ 0x55be12847440] left block unavailable for requested intr", "last_exit_at": "2026-07-02T15:01:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 194.069, "last_started_at": "2026-07-02T14:58:30+09:00", "pid": 11134, "process": {"exists": false, "pid": 11134}, "restart_count": 42, "restart_count_per_hour": 4, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 43}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.69, "sample_count": 2227, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 172254, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T15:01:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-02T15:01:44+09:00", "last_frame_read_age_sec": 0.086, "last_frame_read_at": "2026-07-02T15:01:44+09:00", "mjpeg": {"age_sec": 0.08, "exists": true, "mtime": "2026-07-02T15:01:44.214190+09:00", "mtime_epoch": 1782972104.21419, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 206989}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T15:01:44+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1365.984, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 115, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.873, "pushed": 3.873, "read": 3.839}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.894, "count": 116, "max_ms": 5.34, "p95_ms": 4.946}, "gstreamer_push": {"avg_ms": 1.308, "count": 116, "max_ms": 1.47, "p95_ms": 1.422}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.521, "count": 116, "max_ms": 79.721, "p95_ms": 68.108}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.722, "count": 116, "max_ms": 103.106, "p95_ms": 83.688}, "yolo11_640_fire_11class": {"avg_ms": 51.49, "count": 116, "max_ms": 84.737, "p95_ms": 70.373}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.831, "count": 116, "max_ms": 91.642, "p95_ms": 80.715}}, "truncated_count": 0}, "processing_total": {"avg_ms": 74.236, "count": 116, "max_ms": 116.064, "p95_ms": 97.498}, "read_interval": {"avg_ms": 260.468, "count": 115, "max_ms": 10119.104, "p95_ms": 114.828}, "triton_wait": {"avg_ms": 61.667, "count": 116, "max_ms": 103.119, "p95_ms": 84.817}, "visualize": {"avg_ms": 3.742, "count": 116, "max_ms": 7.711, "p95_ms": 4.344}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.954, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 1435}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5344.734, "limit_mb": 8796093022207.996}, "detail": {"pid": 11241, "restart_count": 43, "start_count": 44}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 32, "h264_decode_error_count": 67, "h264_decode_error_count_per_hour": 20, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T15:01:44+09:00", "last_error_summary": "[h264 @ 0x55be11e2b180] cabac decode of qscale diff failed at 24 40 [h264 @ 0x55be11e2b180] error while decoding MB 24 40, bytestream 5238 [h264 @ 0x55be129e5080] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55be129e5080] error while decoding MB 0 32, bytestream 1842 [h264 @ 0x55be12916200] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55be12916200] error while decoding MB 0 22, bytestream 5628 [h264 @ 0x55be12847440] left block unavailable for requested intr", "last_exit_at": "2026-07-02T15:01:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T15:01:46+09:00", "pid": 11241, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.633, "name": "ffmpeg", "pid": 11241, "rss_mb": 136.633, "state": "S (sleeping)", "threads": 106}, "restart_count": 43, "restart_count_per_hour": 5, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 44}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 97.712, "sample_count": 2177, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 172254, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T15:01:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.319, "last_frame_pushed_at": "2026-07-02T15:01:44+09:00", "last_frame_read_age_sec": 4.39, "last_frame_read_at": "2026-07-02T15:01:44+09:00", "mjpeg": {"age_sec": 4.301, "exists": true, "mtime": "2026-07-02T15:01:44.298192+09:00", "mtime_epoch": 1782972104.2981915, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 207787}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T15:01:48+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1365.984, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 71, "pushed_frames": 72, "read_frames": 71, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.367, "pushed": 2.4, "read": 2.367}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.96, "count": 71, "max_ms": 5.34, "p95_ms": 5.159}, "gstreamer_push": {"avg_ms": 1.307, "count": 72, "max_ms": 1.47, "p95_ms": 1.426}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.96, "count": 71, "max_ms": 75.326, "p95_ms": 67.976}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.709, "count": 71, "max_ms": 103.106, "p95_ms": 84.555}, "yolo11_640_fire_11class": {"avg_ms": 51.672, "count": 71, "max_ms": 74.555, "p95_ms": 71.314}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.092, "count": 71, "max_ms": 90.421, "p95_ms": 79.41}}, "truncated_count": 0}, "processing_total": {"avg_ms": 74.438, "count": 71, "max_ms": 116.064, "p95_ms": 96.975}, "read_interval": {"avg_ms": 361.607, "count": 71, "max_ms": 10119.104, "p95_ms": 114.828}, "triton_wait": {"avg_ms": 61.718, "count": 71, "max_ms": 103.119, "p95_ms": 84.567}, "visualize": {"avg_ms": 3.801, "count": 71, "max_ms": 7.711, "p95_ms": 4.488}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 1423}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5188.797, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x560c74119900] error while decoding MB 39 59, bytestream -23 [in#0/rtsp @ 0x560c73736e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x560c73736e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 28, "h264_decode_error_count": 28, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T15:03:28+09:00", "last_error_summary": "[h264 @ 0x560c74119900] error while decoding MB 39 59, bytestream -23 [in#0/rtsp @ 0x560c73736e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x560c73736e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T14:52:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 687.786, "last_started_at": "2026-07-02T14:52:00+09:00", "pid": 10875, "process": {"exists": false, "pid": 10875}, "restart_count": 27, "restart_count_per_hour": 2, "rtsp_timeout_count": 38, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 28}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.958, "sample_count": 2501, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 170783, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T15:03:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.076, "last_frame_pushed_at": "2026-07-02T15:03:28+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-02T15:03:28+09:00", "mjpeg": {"age_sec": 0.062, "exists": true, "mtime": "2026-07-02T15:03:28.700204+09:00", "mtime_epoch": 1782972208.7002037, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 234015}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T15:03:28+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1113.723, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.631, "pushed": 0.631, "read": 0.684}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.86, "count": 13, "max_ms": 9.811, "p95_ms": 9.811}, "gstreamer_push": {"avg_ms": 1.315, "count": 12, "max_ms": 1.471, "p95_ms": 1.471}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.819, "count": 13, "max_ms": 63.554, "p95_ms": 63.554}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.262, "count": 13, "max_ms": 59.032, "p95_ms": 59.032}, "yolo11_640_fire_11class": {"avg_ms": 51.068, "count": 13, "max_ms": 66.878, "p95_ms": 66.878}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.976, "count": 13, "max_ms": 75.008, "p95_ms": 75.008}}, "truncated_count": 0}, "processing_total": {"avg_ms": 67.871, "count": 12, "max_ms": 89.29, "p95_ms": 89.29}, "read_interval": {"avg_ms": 3048.1, "count": 13, "max_ms": 20678.644, "p95_ms": 20678.644}, "triton_wait": {"avg_ms": 52.156, "count": 13, "max_ms": 75.019, "p95_ms": 75.019}, "visualize": {"avg_ms": 3.821, "count": 13, "max_ms": 4.534, "p95_ms": 4.534}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 19.011, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 1423}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5188.656, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T15:03:28+09:00", "last_error_summary": "[h264 @ 0x560c74119900] error while decoding MB 39 59, bytestream -23 [in#0/rtsp @ 0x560c73736e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x560c73736e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 28, "h264_decode_error_count": 28, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T15:03:28+09:00", "last_error_summary": "[h264 @ 0x560c74119900] error while decoding MB 39 59, bytestream -23 [in#0/rtsp @ 0x560c73736e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x560c73736e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T15:03:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 687.802, "last_started_at": "2026-07-02T14:52:00+09:00", "pid": 10875, "process": {"exists": false, "pid": 10875}, "restart_count": 27, "restart_count_per_hour": 2, "rtsp_timeout_count": 38, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 28}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.958, "sample_count": 2502, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 170783, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T15:03:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T15:03:28+09:00", "last_frame_read_age_sec": 0.08, "last_frame_read_at": "2026-07-02T15:03:28+09:00", "mjpeg": {"age_sec": 0.078, "exists": true, "mtime": "2026-07-02T15:03:28.700204+09:00", "mtime_epoch": 1782972208.7002037, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 234015}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T15:03:28+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1113.723, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.683, "pushed": 0.683, "read": 0.683}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.86, "count": 13, "max_ms": 9.811, "p95_ms": 9.811}, "gstreamer_push": {"avg_ms": 1.325, "count": 13, "max_ms": 1.471, "p95_ms": 1.471}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.819, "count": 13, "max_ms": 63.554, "p95_ms": 63.554}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.262, "count": 13, "max_ms": 59.032, "p95_ms": 59.032}, "yolo11_640_fire_11class": {"avg_ms": 51.068, "count": 13, "max_ms": 66.878, "p95_ms": 66.878}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.976, "count": 13, "max_ms": 75.008, "p95_ms": 75.008}}, "truncated_count": 0}, "processing_total": {"avg_ms": 67.992, "count": 13, "max_ms": 89.29, "p95_ms": 89.29}, "read_interval": {"avg_ms": 3048.1, "count": 13, "max_ms": 20678.644, "p95_ms": 20678.644}, "triton_wait": {"avg_ms": 52.156, "count": 13, "max_ms": 75.019, "p95_ms": 75.019}, "visualize": {"avg_ms": 3.821, "count": 13, "max_ms": 4.534, "p95_ms": 4.534}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 19.027, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 1423}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5188.512, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 28, "h264_decode_error_count": 28, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T15:03:28+09:00", "last_error_summary": "[h264 @ 0x560c74119900] error while decoding MB 39 59, bytestream -23 [in#0/rtsp @ 0x560c73736e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x560c73736e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T15:03:28+09:00", "last_returncode": 0, "last_stable_duration_sec": 687.81, "last_started_at": "2026-07-02T14:52:00+09:00", "pid": 10875, "process": {"exists": false, "pid": 10875}, "restart_count": 27, "restart_count_per_hour": 2, "rtsp_timeout_count": 38, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 28}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.958, "sample_count": 2502, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 170783, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T15:03:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-02T15:03:28+09:00", "last_frame_read_age_sec": 0.087, "last_frame_read_at": "2026-07-02T15:03:28+09:00", "mjpeg": {"age_sec": 0.085, "exists": true, "mtime": "2026-07-02T15:03:28.700204+09:00", "mtime_epoch": 1782972208.7002037, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 234015}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T15:03:28+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1113.723, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.683, "pushed": 0.683, "read": 0.683}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.86, "count": 13, "max_ms": 9.811, "p95_ms": 9.811}, "gstreamer_push": {"avg_ms": 1.325, "count": 13, "max_ms": 1.471, "p95_ms": 1.471}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.819, "count": 13, "max_ms": 63.554, "p95_ms": 63.554}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.262, "count": 13, "max_ms": 59.032, "p95_ms": 59.032}, "yolo11_640_fire_11class": {"avg_ms": 51.068, "count": 13, "max_ms": 66.878, "p95_ms": 66.878}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.976, "count": 13, "max_ms": 75.008, "p95_ms": 75.008}}, "truncated_count": 0}, "processing_total": {"avg_ms": 67.992, "count": 13, "max_ms": 89.29, "p95_ms": 89.29}, "read_interval": {"avg_ms": 3048.1, "count": 13, "max_ms": 20678.644, "p95_ms": 20678.644}, "triton_wait": {"avg_ms": 52.156, "count": 13, "max_ms": 75.019, "p95_ms": 75.019}, "visualize": {"avg_ms": 3.821, "count": 13, "max_ms": 4.534, "p95_ms": 4.534}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 19.034, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 1423}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5212.82, "limit_mb": 8796093022207.996}, "detail": {"pid": 11348, "restart_count": 28, "start_count": 29}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 28, "h264_decode_error_count": 28, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T15:03:28+09:00", "last_error_summary": "[h264 @ 0x560c74119900] error while decoding MB 39 59, bytestream -23 [in#0/rtsp @ 0x560c73736e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x560c73736e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T15:03:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T15:03:30+09:00", "pid": 11348, "process": {"exists": true, "fd_count": 4, "hwm_mb": 57.68, "name": "ffmpeg", "pid": 11348, "rss_mb": 57.68, "state": "S (sleeping)", "threads": 1}, "restart_count": 28, "restart_count_per_hour": 3, "rtsp_timeout_count": 38, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 29}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.958, "sample_count": 2492, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 170783, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T15:03:28+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 4.222, "last_frame_pushed_at": "2026-07-02T15:03:28+09:00", "last_frame_read_age_sec": 4.295, "last_frame_read_at": "2026-07-02T15:03:28+09:00", "mjpeg": {"age_sec": 4.204, "exists": true, "mtime": "2026-07-02T15:03:28.788205+09:00", "mtime_epoch": 1782972208.7882054, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 233708}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T15:03:32+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1113.723, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.559, "pushed": 0.559, "read": 0.559}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.86, "count": 13, "max_ms": 9.811, "p95_ms": 9.811}, "gstreamer_push": {"avg_ms": 1.325, "count": 13, "max_ms": 1.471, "p95_ms": 1.471}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.819, "count": 13, "max_ms": 63.554, "p95_ms": 63.554}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 49.262, "count": 13, "max_ms": 59.032, "p95_ms": 59.032}, "yolo11_640_fire_11class": {"avg_ms": 51.068, "count": 13, "max_ms": 66.878, "p95_ms": 66.878}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 49.976, "count": 13, "max_ms": 75.008, "p95_ms": 75.008}}, "truncated_count": 0}, "processing_total": {"avg_ms": 67.992, "count": 13, "max_ms": 89.29, "p95_ms": 89.29}, "read_interval": {"avg_ms": 3048.1, "count": 13, "max_ms": 20678.644, "p95_ms": 20678.644}, "triton_wait": {"avg_ms": 52.156, "count": 13, "max_ms": 75.019, "p95_ms": 75.019}, "visualize": {"avg_ms": 3.821, "count": 13, "max_ms": 4.534, "p95_ms": 4.534}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 23.242, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 1445}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5251.812, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55ab56f10840] error while decoding MB 48 36, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 33, "h264_decode_error_count": 68, "h264_decode_error_count_per_hour": 21, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T15:04:38+09:00", "last_error_summary": "[h264 @ 0x55ab56f10840] error while decoding MB 48 36, bytestream -5", "last_exit_at": "2026-07-02T15:01:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 171.489, "last_started_at": "2026-07-02T15:01:46+09:00", "pid": 11241, "process": {"exists": false, "pid": 11241}, "restart_count": 43, "restart_count_per_hour": 5, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 44}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 96.639, "sample_count": 2091, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 173458, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T15:04:37+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.072, "last_frame_pushed_at": "2026-07-02T15:04:38+09:00", "last_frame_read_age_sec": 0.058, "last_frame_read_at": "2026-07-02T15:04:38+09:00", "mjpeg": {"age_sec": 0.057, "exists": true, "mtime": "2026-07-02T15:04:38.029520+09:00", "mtime_epoch": 1782972278.0295196, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199431}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T15:04:38+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1366.5, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 244, "pushed_frames": 244, "read_frames": 245, "select_timeouts": 7, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.182, "pushed": 8.182, "read": 8.215}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.825, "count": 244, "max_ms": 5.819, "p95_ms": 4.45}, "gstreamer_push": {"avg_ms": 1.335, "count": 244, "max_ms": 2.055, "p95_ms": 1.457}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.18, "count": 244, "max_ms": 88.211, "p95_ms": 69.238}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.572, "count": 244, "max_ms": 104.797, "p95_ms": 84.323}, "yolo11_640_fire_11class": {"avg_ms": 54.864, "count": 244, "max_ms": 88.199, "p95_ms": 74.758}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.48, "count": 244, "max_ms": 92.47, "p95_ms": 78.672}}, "truncated_count": 0}, "processing_total": {"avg_ms": 73.818, "count": 244, "max_ms": 117.157, "p95_ms": 100.885}, "read_interval": {"avg_ms": 131.715, "count": 245, "max_ms": 3089.85, "p95_ms": 120.73}, "triton_wait": {"avg_ms": 61.248, "count": 244, "max_ms": 104.809, "p95_ms": 88.46}, "visualize": {"avg_ms": 3.73, "count": 244, "max_ms": 5.615, "p95_ms": 4.221}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.823, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 1445}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5251.43, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T15:04:38+09:00", "last_error_summary": "[h264 @ 0x55ab56f10840] error while decoding MB 48 36, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 33, "h264_decode_error_count": 68, "h264_decode_error_count_per_hour": 21, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T15:04:38+09:00", "last_error_summary": "[h264 @ 0x55ab56f10840] error while decoding MB 48 36, bytestream -5", "last_exit_at": "2026-07-02T15:04:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 171.54, "last_started_at": "2026-07-02T15:01:46+09:00", "pid": 11241, "process": {"exists": false, "pid": 11241}, "restart_count": 43, "restart_count_per_hour": 5, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 44}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 96.639, "sample_count": 2092, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 173459, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T15:04:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T15:04:38+09:00", "last_frame_read_age_sec": 0.11, "last_frame_read_at": "2026-07-02T15:04:38+09:00", "mjpeg": {"age_sec": 0.108, "exists": true, "mtime": "2026-07-02T15:04:38.029520+09:00", "mtime_epoch": 1782972278.0295196, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199431}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T15:04:38+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1366.5, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 245, "pushed_frames": 245, "read_frames": 245, "select_timeouts": 7, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.201, "pushed": 8.201, "read": 8.201}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.826, "count": 245, "max_ms": 5.819, "p95_ms": 4.45}, "gstreamer_push": {"avg_ms": 1.335, "count": 245, "max_ms": 2.055, "p95_ms": 1.457}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.172, "count": 245, "max_ms": 88.211, "p95_ms": 69.238}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.687, "count": 245, "max_ms": 104.797, "p95_ms": 87.708}, "yolo11_640_fire_11class": {"avg_ms": 54.871, "count": 245, "max_ms": 88.199, "p95_ms": 74.758}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.475, "count": 245, "max_ms": 92.47, "p95_ms": 78.672}}, "truncated_count": 0}, "processing_total": {"avg_ms": 73.926, "count": 245, "max_ms": 117.157, "p95_ms": 100.885}, "read_interval": {"avg_ms": 131.715, "count": 245, "max_ms": 3089.85, "p95_ms": 120.73}, "triton_wait": {"avg_ms": 61.356, "count": 245, "max_ms": 104.809, "p95_ms": 88.46}, "visualize": {"avg_ms": 3.73, "count": 245, "max_ms": 5.615, "p95_ms": 4.221}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.874, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 1445}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5251.418, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 33, "h264_decode_error_count": 68, "h264_decode_error_count_per_hour": 21, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T15:04:38+09:00", "last_error_summary": "[h264 @ 0x55ab56f10840] error while decoding MB 48 36, bytestream -5", "last_exit_at": "2026-07-02T15:04:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 171.548, "last_started_at": "2026-07-02T15:01:46+09:00", "pid": 11241, "process": {"exists": false, "pid": 11241}, "restart_count": 43, "restart_count_per_hour": 5, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 44}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 96.639, "sample_count": 2092, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 173459, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T15:04:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.015, "last_frame_pushed_at": "2026-07-02T15:04:38+09:00", "last_frame_read_age_sec": 0.118, "last_frame_read_at": "2026-07-02T15:04:38+09:00", "mjpeg": {"age_sec": 0.116, "exists": true, "mtime": "2026-07-02T15:04:38.029520+09:00", "mtime_epoch": 1782972278.0295196, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199431}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T15:04:38+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1366.5, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 245, "pushed_frames": 245, "read_frames": 245, "select_timeouts": 7, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.199, "pushed": 8.199, "read": 8.199}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.826, "count": 245, "max_ms": 5.819, "p95_ms": 4.45}, "gstreamer_push": {"avg_ms": 1.335, "count": 245, "max_ms": 2.055, "p95_ms": 1.457}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.172, "count": 245, "max_ms": 88.211, "p95_ms": 69.238}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.687, "count": 245, "max_ms": 104.797, "p95_ms": 87.708}, "yolo11_640_fire_11class": {"avg_ms": 54.871, "count": 245, "max_ms": 88.199, "p95_ms": 74.758}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.475, "count": 245, "max_ms": 92.47, "p95_ms": 78.672}}, "truncated_count": 0}, "processing_total": {"avg_ms": 73.926, "count": 245, "max_ms": 117.157, "p95_ms": 100.885}, "read_interval": {"avg_ms": 131.715, "count": 245, "max_ms": 3089.85, "p95_ms": 120.73}, "triton_wait": {"avg_ms": 61.356, "count": 245, "max_ms": 104.809, "p95_ms": 88.46}, "visualize": {"avg_ms": 3.73, "count": 245, "max_ms": 5.615, "p95_ms": 4.221}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.882, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 41, "tracking_summary": 1445}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5357.73, "limit_mb": 8796093022207.996}, "detail": {"pid": 11500, "restart_count": 44, "start_count": 45}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 33, "h264_decode_error_count": 68, "h264_decode_error_count_per_hour": 21, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T15:04:38+09:00", "last_error_summary": "[h264 @ 0x55ab56f10840] error while decoding MB 48 36, bytestream -5", "last_exit_at": "2026-07-02T15:04:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T15:04:40+09:00", "pid": 11500, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.453, "name": "ffmpeg", "pid": 11500, "rss_mb": 136.453, "state": "S (sleeping)", "threads": 106}, "restart_count": 44, "restart_count_per_hour": 6, "rtsp_timeout_count": 18, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 45}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 96.528, "sample_count": 2053, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 173459, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T15:04:38+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.416, "last_frame_pushed_at": "2026-07-02T15:04:38+09:00", "last_frame_read_age_sec": 4.518, "last_frame_read_at": "2026-07-02T15:04:38+09:00", "mjpeg": {"age_sec": 4.401, "exists": true, "mtime": "2026-07-02T15:04:38.145522+09:00", "mtime_epoch": 1782972278.1455216, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 199406}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T15:04:42+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1366.5, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 197, "pushed_frames": 197, "read_frames": 196, "select_timeouts": 7, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.571, "pushed": 6.571, "read": 6.538}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.842, "count": 197, "max_ms": 5.819, "p95_ms": 4.527}, "gstreamer_push": {"avg_ms": 1.337, "count": 197, "max_ms": 2.055, "p95_ms": 1.457}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.378, "count": 197, "max_ms": 78.918, "p95_ms": 69.484}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.126, "count": 197, "max_ms": 104.797, "p95_ms": 88.446}, "yolo11_640_fire_11class": {"avg_ms": 55.14, "count": 197, "max_ms": 83.277, "p95_ms": 74.855}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.837, "count": 197, "max_ms": 84.359, "p95_ms": 78.672}}, "truncated_count": 0}, "processing_total": {"avg_ms": 74.529, "count": 197, "max_ms": 117.157, "p95_ms": 100.885}, "read_interval": {"avg_ms": 130.351, "count": 196, "max_ms": 3089.85, "p95_ms": 120.482}, "triton_wait": {"avg_ms": 61.914, "count": 197, "max_ms": 104.809, "p95_ms": 88.46}, "visualize": {"avg_ms": 3.738, "count": 197, "max_ms": 5.615, "p95_ms": 4.288}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.979, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 1447}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5207.422, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55e7f7eb4240] error while decoding MB 72 53, bytestream -15 [in#0/rtsp @ 0x55e7f7d1fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e7f7d1fe00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 29, "h264_decode_error_count": 29, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T15:09:30+09:00", "last_error_summary": "[h264 @ 0x55e7f7eb4240] error while decoding MB 72 53, bytestream -15 [in#0/rtsp @ 0x55e7f7d1fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e7f7d1fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T15:03:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 359.276, "last_started_at": "2026-07-02T15:03:30+09:00", "pid": 11348, "process": {"exists": false, "pid": 11348}, "restart_count": 28, "restart_count_per_hour": 3, "rtsp_timeout_count": 40, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 29}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 146.888, "sample_count": 2408, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 173752, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T15:09:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.073, "last_frame_pushed_at": "2026-07-02T15:09:30+09:00", "last_frame_read_age_sec": 0.061, "last_frame_read_at": "2026-07-02T15:09:30+09:00", "mjpeg": {"age_sec": 0.053, "exists": true, "mtime": "2026-07-02T15:09:30.214932+09:00", "mtime_epoch": 1782972570.2149322, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 247845}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T15:09:30+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1115.012, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.604, "pushed": 0.604, "read": 0.654}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.767, "count": 13, "max_ms": 9.745, "p95_ms": 9.745}, "gstreamer_push": {"avg_ms": 1.301, "count": 12, "max_ms": 1.456, "p95_ms": 1.456}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.631, "count": 13, "max_ms": 53.451, "p95_ms": 53.451}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.522, "count": 13, "max_ms": 72.457, "p95_ms": 72.457}, "yolo11_640_fire_11class": {"avg_ms": 52.65, "count": 13, "max_ms": 55.908, "p95_ms": 55.908}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.832, "count": 13, "max_ms": 53.813, "p95_ms": 53.813}}, "truncated_count": 0}, "processing_total": {"avg_ms": 70.756, "count": 12, "max_ms": 87.076, "p95_ms": 87.076}, "read_interval": {"avg_ms": 2741.043, "count": 13, "max_ms": 18881.723, "p95_ms": 18881.723}, "triton_wait": {"avg_ms": 54.65, "count": 13, "max_ms": 72.47, "p95_ms": 72.47}, "visualize": {"avg_ms": 3.906, "count": 13, "max_ms": 5.134, "p95_ms": 5.134}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 19.874, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 1447}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5207.125, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T15:09:30+09:00", "last_error_summary": "[h264 @ 0x55e7f7eb4240] error while decoding MB 72 53, bytestream -15 [in#0/rtsp @ 0x55e7f7d1fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e7f7d1fe00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 29, "h264_decode_error_count": 29, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T15:09:30+09:00", "last_error_summary": "[h264 @ 0x55e7f7eb4240] error while decoding MB 72 53, bytestream -15 [in#0/rtsp @ 0x55e7f7d1fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e7f7d1fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T15:09:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 359.293, "last_started_at": "2026-07-02T15:03:30+09:00", "pid": 11348, "process": {"exists": false, "pid": 11348}, "restart_count": 28, "restart_count_per_hour": 3, "rtsp_timeout_count": 40, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 29}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 146.888, "sample_count": 2408, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 173752, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T15:09:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T15:09:30+09:00", "last_frame_read_age_sec": 0.078, "last_frame_read_at": "2026-07-02T15:09:30+09:00", "mjpeg": {"age_sec": 0.07, "exists": true, "mtime": "2026-07-02T15:09:30.214932+09:00", "mtime_epoch": 1782972570.2149322, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 247845}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T15:09:30+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1115.012, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.654, "pushed": 0.654, "read": 0.654}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.767, "count": 13, "max_ms": 9.745, "p95_ms": 9.745}, "gstreamer_push": {"avg_ms": 1.3, "count": 13, "max_ms": 1.456, "p95_ms": 1.456}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.631, "count": 13, "max_ms": 53.451, "p95_ms": 53.451}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.522, "count": 13, "max_ms": 72.457, "p95_ms": 72.457}, "yolo11_640_fire_11class": {"avg_ms": 52.65, "count": 13, "max_ms": 55.908, "p95_ms": 55.908}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.832, "count": 13, "max_ms": 53.813, "p95_ms": 53.813}}, "truncated_count": 0}, "processing_total": {"avg_ms": 70.725, "count": 13, "max_ms": 87.076, "p95_ms": 87.076}, "read_interval": {"avg_ms": 2741.043, "count": 13, "max_ms": 18881.723, "p95_ms": 18881.723}, "triton_wait": {"avg_ms": 54.65, "count": 13, "max_ms": 72.47, "p95_ms": 72.47}, "visualize": {"avg_ms": 3.906, "count": 13, "max_ms": 5.134, "p95_ms": 5.134}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 19.891, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 1447}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5207.117, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 29, "h264_decode_error_count": 29, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T15:09:30+09:00", "last_error_summary": "[h264 @ 0x55e7f7eb4240] error while decoding MB 72 53, bytestream -15 [in#0/rtsp @ 0x55e7f7d1fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e7f7d1fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T15:09:30+09:00", "last_returncode": 0, "last_stable_duration_sec": 359.301, "last_started_at": "2026-07-02T15:03:30+09:00", "pid": 11348, "process": {"exists": false, "pid": 11348}, "restart_count": 28, "restart_count_per_hour": 3, "rtsp_timeout_count": 40, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 29}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 146.888, "sample_count": 2408, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 173752, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T15:09:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-02T15:09:30+09:00", "last_frame_read_age_sec": 0.086, "last_frame_read_at": "2026-07-02T15:09:30+09:00", "mjpeg": {"age_sec": 0.078, "exists": true, "mtime": "2026-07-02T15:09:30.214932+09:00", "mtime_epoch": 1782972570.2149322, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 247845}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T15:09:30+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1115.012, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 28, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.653, "pushed": 0.653, "read": 0.653}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.767, "count": 13, "max_ms": 9.745, "p95_ms": 9.745}, "gstreamer_push": {"avg_ms": 1.3, "count": 13, "max_ms": 1.456, "p95_ms": 1.456}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.631, "count": 13, "max_ms": 53.451, "p95_ms": 53.451}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.522, "count": 13, "max_ms": 72.457, "p95_ms": 72.457}, "yolo11_640_fire_11class": {"avg_ms": 52.65, "count": 13, "max_ms": 55.908, "p95_ms": 55.908}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.832, "count": 13, "max_ms": 53.813, "p95_ms": 53.813}}, "truncated_count": 0}, "processing_total": {"avg_ms": 70.725, "count": 13, "max_ms": 87.076, "p95_ms": 87.076}, "read_interval": {"avg_ms": 2741.043, "count": 13, "max_ms": 18881.723, "p95_ms": 18881.723}, "triton_wait": {"avg_ms": 54.65, "count": 13, "max_ms": 72.47, "p95_ms": 72.47}, "visualize": {"avg_ms": 3.906, "count": 13, "max_ms": 5.134, "p95_ms": 5.134}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 19.899, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 1447}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5327.902, "limit_mb": 8796093022207.996}, "detail": {"pid": 11607, "restart_count": 29, "start_count": 30}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 29, "h264_decode_error_count": 29, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T15:09:30+09:00", "last_error_summary": "[h264 @ 0x55e7f7eb4240] error while decoding MB 72 53, bytestream -15 [in#0/rtsp @ 0x55e7f7d1fe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e7f7d1fe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T15:09:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T15:09:32+09:00", "pid": 11607, "process": {"exists": true, "fd_count": 5, "hwm_mb": 156.609, "name": "ffmpeg", "pid": 11607, "rss_mb": 156.445, "state": "R (running)", "threads": 49}, "restart_count": 29, "restart_count_per_hour": 4, "rtsp_timeout_count": 40, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 30}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 147.731, "sample_count": 2370, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 173752, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T15:09:30+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 4.16, "last_frame_pushed_at": "2026-07-02T15:09:30+09:00", "last_frame_read_age_sec": 4.232, "last_frame_read_at": "2026-07-02T15:09:30+09:00", "mjpeg": {"age_sec": 4.14, "exists": true, "mtime": "2026-07-02T15:09:30.298934+09:00", "mtime_epoch": 1782972570.2989337, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 247771}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T15:09:34+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1115.012, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.541, "pushed": 0.541, "read": 0.541}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.767, "count": 13, "max_ms": 9.745, "p95_ms": 9.745}, "gstreamer_push": {"avg_ms": 1.3, "count": 13, "max_ms": 1.456, "p95_ms": 1.456}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.631, "count": 13, "max_ms": 53.451, "p95_ms": 53.451}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.522, "count": 13, "max_ms": 72.457, "p95_ms": 72.457}, "yolo11_640_fire_11class": {"avg_ms": 52.65, "count": 13, "max_ms": 55.908, "p95_ms": 55.908}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 50.832, "count": 13, "max_ms": 53.813, "p95_ms": 53.813}}, "truncated_count": 0}, "processing_total": {"avg_ms": 70.725, "count": 13, "max_ms": 87.076, "p95_ms": 87.076}, "read_interval": {"avg_ms": 2741.043, "count": 13, "max_ms": 18881.723, "p95_ms": 18881.723}, "triton_wait": {"avg_ms": 54.65, "count": 13, "max_ms": 72.47, "p95_ms": 72.47}, "visualize": {"avg_ms": 3.906, "count": 13, "max_ms": 5.134, "p95_ms": 5.134}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 24.045, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 1624}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5243.164, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x558fe637e000] error while decoding MB 0 62, bytestream -33 [in#0/rtsp @ 0x558fe61eee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558fe61eee00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 30, "h264_decode_error_count": 30, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T15:45:41+09:00", "last_error_summary": "[h264 @ 0x558fe637e000] error while decoding MB 0 62, bytestream -33 [in#0/rtsp @ 0x558fe61eee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558fe61eee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T15:09:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 2168.633, "last_started_at": "2026-07-02T15:09:32+09:00", "pid": 11607, "process": {"exists": false, "pid": 11607}, "restart_count": 29, "restart_count_per_hour": 3, "rtsp_timeout_count": 42, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 30}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 142.269, "sample_count": 2572, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 194991, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T15:45:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.073, "last_frame_pushed_at": "2026-07-02T15:45:40+09:00", "last_frame_read_age_sec": 0.062, "last_frame_read_at": "2026-07-02T15:45:41+09:00", "mjpeg": {"age_sec": 0.043, "exists": true, "mtime": "2026-07-02T15:45:41.028606+09:00", "mtime_epoch": 1782974741.0286064, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 236068}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T15:45:41+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1125.066, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.691, "pushed": 0.691, "read": 0.749}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.842, "count": 13, "max_ms": 8.576, "p95_ms": 8.576}, "gstreamer_push": {"avg_ms": 1.32, "count": 12, "max_ms": 1.43, "p95_ms": 1.43}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.71, "count": 13, "max_ms": 69.44, "p95_ms": 69.44}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.943, "count": 13, "max_ms": 90.11, "p95_ms": 90.11}, "yolo11_640_fire_11class": {"avg_ms": 56.445, "count": 13, "max_ms": 70.58, "p95_ms": 70.58}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.564, "count": 13, "max_ms": 71.516, "p95_ms": 71.516}}, "truncated_count": 0}, "processing_total": {"avg_ms": 82.504, "count": 12, "max_ms": 107.091, "p95_ms": 107.091}, "read_interval": {"avg_ms": 2457.406, "count": 13, "max_ms": 16210.045, "p95_ms": 16210.045}, "triton_wait": {"avg_ms": 65.444, "count": 13, "max_ms": 90.128, "p95_ms": 90.128}, "visualize": {"avg_ms": 3.849, "count": 13, "max_ms": 4.776, "p95_ms": 4.776}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.357, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 1624}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5243.023, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T15:45:41+09:00", "last_error_summary": "[h264 @ 0x558fe637e000] error while decoding MB 0 62, bytestream -33 [in#0/rtsp @ 0x558fe61eee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558fe61eee00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 30, "h264_decode_error_count": 30, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T15:45:41+09:00", "last_error_summary": "[h264 @ 0x558fe637e000] error while decoding MB 0 62, bytestream -33 [in#0/rtsp @ 0x558fe61eee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558fe61eee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T15:45:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 2168.654, "last_started_at": "2026-07-02T15:09:32+09:00", "pid": 11607, "process": {"exists": false, "pid": 11607}, "restart_count": 29, "restart_count_per_hour": 3, "rtsp_timeout_count": 42, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 30}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 142.269, "sample_count": 2573, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 194991, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T15:45:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-02T15:45:41+09:00", "last_frame_read_age_sec": 0.083, "last_frame_read_at": "2026-07-02T15:45:41+09:00", "mjpeg": {"age_sec": 0.064, "exists": true, "mtime": "2026-07-02T15:45:41.028606+09:00", "mtime_epoch": 1782974741.0286064, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 236068}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T15:45:41+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1125.066, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.748, "pushed": 0.748, "read": 0.748}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.842, "count": 13, "max_ms": 8.576, "p95_ms": 8.576}, "gstreamer_push": {"avg_ms": 1.315, "count": 13, "max_ms": 1.43, "p95_ms": 1.43}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.71, "count": 13, "max_ms": 69.44, "p95_ms": 69.44}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.943, "count": 13, "max_ms": 90.11, "p95_ms": 90.11}, "yolo11_640_fire_11class": {"avg_ms": 56.445, "count": 13, "max_ms": 70.58, "p95_ms": 70.58}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.564, "count": 13, "max_ms": 71.516, "p95_ms": 71.516}}, "truncated_count": 0}, "processing_total": {"avg_ms": 81.672, "count": 13, "max_ms": 107.091, "p95_ms": 107.091}, "read_interval": {"avg_ms": 2457.406, "count": 13, "max_ms": 16210.045, "p95_ms": 16210.045}, "triton_wait": {"avg_ms": 65.444, "count": 13, "max_ms": 90.128, "p95_ms": 90.128}, "visualize": {"avg_ms": 3.849, "count": 13, "max_ms": 4.776, "p95_ms": 4.776}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.378, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 1624}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5243.016, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 30, "h264_decode_error_count": 30, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T15:45:41+09:00", "last_error_summary": "[h264 @ 0x558fe637e000] error while decoding MB 0 62, bytestream -33 [in#0/rtsp @ 0x558fe61eee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558fe61eee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T15:45:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 2168.661, "last_started_at": "2026-07-02T15:09:32+09:00", "pid": 11607, "process": {"exists": false, "pid": 11607}, "restart_count": 29, "restart_count_per_hour": 3, "rtsp_timeout_count": 42, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 30}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 142.269, "sample_count": 2573, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 194991, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T15:45:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-02T15:45:41+09:00", "last_frame_read_age_sec": 0.09, "last_frame_read_at": "2026-07-02T15:45:41+09:00", "mjpeg": {"age_sec": 0.071, "exists": true, "mtime": "2026-07-02T15:45:41.028606+09:00", "mtime_epoch": 1782974741.0286064, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 236068}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T15:45:41+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1125.066, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.748, "pushed": 0.748, "read": 0.748}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.842, "count": 13, "max_ms": 8.576, "p95_ms": 8.576}, "gstreamer_push": {"avg_ms": 1.315, "count": 13, "max_ms": 1.43, "p95_ms": 1.43}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.71, "count": 13, "max_ms": 69.44, "p95_ms": 69.44}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.943, "count": 13, "max_ms": 90.11, "p95_ms": 90.11}, "yolo11_640_fire_11class": {"avg_ms": 56.445, "count": 13, "max_ms": 70.58, "p95_ms": 70.58}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.564, "count": 13, "max_ms": 71.516, "p95_ms": 71.516}}, "truncated_count": 0}, "processing_total": {"avg_ms": 81.672, "count": 13, "max_ms": 107.091, "p95_ms": 107.091}, "read_interval": {"avg_ms": 2457.406, "count": 13, "max_ms": 16210.045, "p95_ms": 16210.045}, "triton_wait": {"avg_ms": 65.444, "count": 13, "max_ms": 90.128, "p95_ms": 90.128}, "visualize": {"avg_ms": 3.849, "count": 13, "max_ms": 4.776, "p95_ms": 4.776}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.385, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 34, "tracking_summary": 1624}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5311.785, "limit_mb": 8796093022207.996}, "detail": {"pid": 11759, "restart_count": 30, "start_count": 31}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 30, "h264_decode_error_count": 30, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T15:45:41+09:00", "last_error_summary": "[h264 @ 0x558fe637e000] error while decoding MB 0 62, bytestream -33 [in#0/rtsp @ 0x558fe61eee00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x558fe61eee00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T15:45:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T15:45:43+09:00", "pid": 11759, "process": {"exists": true, "fd_count": 5, "hwm_mb": 95.547, "name": "ffmpeg", "pid": 11759, "rss_mb": 95.547, "state": "S (sleeping)", "threads": 19}, "restart_count": 30, "restart_count_per_hour": 4, "rtsp_timeout_count": 42, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 31}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 142.573, "sample_count": 2532, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 194991, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T15:45:41+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 4.243, "last_frame_pushed_at": "2026-07-02T15:45:41+09:00", "last_frame_read_age_sec": 4.317, "last_frame_read_at": "2026-07-02T15:45:41+09:00", "mjpeg": {"age_sec": 4.226, "exists": true, "mtime": "2026-07-02T15:45:41.100608+09:00", "mtime_epoch": 1782974741.1006076, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 235597}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T15:45:45+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1125.066, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.602, "pushed": 0.602, "read": 0.602}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.842, "count": 13, "max_ms": 8.576, "p95_ms": 8.576}, "gstreamer_push": {"avg_ms": 1.315, "count": 13, "max_ms": 1.43, "p95_ms": 1.43}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.71, "count": 13, "max_ms": 69.44, "p95_ms": 69.44}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.943, "count": 13, "max_ms": 90.11, "p95_ms": 90.11}, "yolo11_640_fire_11class": {"avg_ms": 56.445, "count": 13, "max_ms": 70.58, "p95_ms": 70.58}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.564, "count": 13, "max_ms": 71.516, "p95_ms": 71.516}}, "truncated_count": 0}, "processing_total": {"avg_ms": 81.672, "count": 13, "max_ms": 107.091, "p95_ms": 107.091}, "read_interval": {"avg_ms": 2457.406, "count": 13, "max_ms": 16210.045, "p95_ms": 16210.045}, "triton_wait": {"avg_ms": 65.444, "count": 13, "max_ms": 90.128, "p95_ms": 90.128}, "visualize": {"avg_ms": 3.849, "count": 13, "max_ms": 4.776, "p95_ms": 4.776}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.612, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 42, "tracking_summary": 1739}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5302.879, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x556c95997ec0] left block unavailable for requested intra mode [h264 @ 0x556c95997ec0] error while decoding MB 0 16, bytestream 3956 [h264 @ 0x556c95b35bc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556c95b35bc0] error while decoding MB 0 21, bytestream 39826 [h264 @ 0x556c94e435c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556c94e435c0] error while decoding MB 0 32, bytestream 40632 [h264 @ 0x556c95997ec0] left block unavailable for requested", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 34, "h264_decode_error_count": 72, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:04:40+09:00", "last_error_summary": "[h264 @ 0x556c95997ec0] left block unavailable for requested intra mode [h264 @ 0x556c95997ec0] error while decoding MB 0 16, bytestream 3956 [h264 @ 0x556c95b35bc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556c95b35bc0] error while decoding MB 0 21, bytestream 39826 [h264 @ 0x556c94e435c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556c94e435c0] error while decoding MB 0 32, bytestream 40632 [h264 @ 0x556c95997ec0] left block unavailable for requested", "last_exit_at": "2026-07-02T15:04:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 3599.732, "last_started_at": "2026-07-02T15:04:40+09:00", "pid": 11500, "process": {"exists": false, "pid": 11500}, "restart_count": 44, "restart_count_per_hour": 1, "rtsp_timeout_count": 20, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 45}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.308, "sample_count": 2268, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 208708, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:04:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.066, "last_frame_pushed_at": "2026-07-02T16:04:40+09:00", "last_frame_read_age_sec": 0.056, "last_frame_read_at": "2026-07-02T16:04:40+09:00", "mjpeg": {"age_sec": 0.056, "exists": true, "mtime": "2026-07-02T16:04:40.221601+09:00", "mtime_epoch": 1782975880.2216012, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194068}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:04:40+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1383.0, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 68, "pushed_frames": 68, "read_frames": 68, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.274, "pushed": 2.274, "read": 2.274}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 7.634, "count": 68, "max_ms": 11.219, "p95_ms": 8.801}, "gstreamer_push": {"avg_ms": 0.991, "count": 68, "max_ms": 1.152, "p95_ms": 1.114}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.981, "count": 68, "max_ms": 70.833, "p95_ms": 65.199}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.765, "count": 68, "max_ms": 109.429, "p95_ms": 87.247}, "yolo11_640_fire_11class": {"avg_ms": 54.038, "count": 68, "max_ms": 78.089, "p95_ms": 68.716}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.727, "count": 68, "max_ms": 85.726, "p95_ms": 80.635}}, "truncated_count": 0}, "processing_total": {"avg_ms": 145.267, "count": 68, "max_ms": 190.243, "p95_ms": 170.324}, "read_interval": {"avg_ms": 441.233, "count": 68, "max_ms": 10130.995, "p95_ms": 194.445}, "triton_wait": {"avg_ms": 64.159, "count": 68, "max_ms": 109.441, "p95_ms": 87.26}, "visualize": {"avg_ms": 6.508, "count": 68, "max_ms": 8.405, "p95_ms": 7.929}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.907, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 42, "tracking_summary": 1739}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5314.102, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T16:04:40+09:00", "last_error_summary": "[h264 @ 0x556c95997ec0] left block unavailable for requested intra mode [h264 @ 0x556c95997ec0] error while decoding MB 0 16, bytestream 3956 [h264 @ 0x556c95b35bc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556c95b35bc0] error while decoding MB 0 21, bytestream 39826 [h264 @ 0x556c94e435c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556c94e435c0] error while decoding MB 0 32, bytestream 40632 [h264 @ 0x556c95997ec0] left block unavailable for requested", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 34, "h264_decode_error_count": 72, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:04:40+09:00", "last_error_summary": "[h264 @ 0x556c95997ec0] left block unavailable for requested intra mode [h264 @ 0x556c95997ec0] error while decoding MB 0 16, bytestream 3956 [h264 @ 0x556c95b35bc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556c95b35bc0] error while decoding MB 0 21, bytestream 39826 [h264 @ 0x556c94e435c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556c94e435c0] error while decoding MB 0 32, bytestream 40632 [h264 @ 0x556c95997ec0] left block unavailable for requested", "last_exit_at": "2026-07-02T16:04:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 3599.841, "last_started_at": "2026-07-02T15:04:40+09:00", "pid": 11500, "process": {"exists": false, "pid": 11500}, "restart_count": 44, "restart_count_per_hour": 1, "rtsp_timeout_count": 20, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 45}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.417, "sample_count": 2268, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 208709, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:04:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T16:04:40+09:00", "last_frame_read_age_sec": 0.165, "last_frame_read_at": "2026-07-02T16:04:40+09:00", "mjpeg": {"age_sec": 0.164, "exists": true, "mtime": "2026-07-02T16:04:40.221601+09:00", "mtime_epoch": 1782975880.2216012, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194068}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:04:40+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1383.0, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 68, "pushed_frames": 68, "read_frames": 67, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.279, "pushed": 2.279, "read": 2.245}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 7.63, "count": 68, "max_ms": 11.219, "p95_ms": 8.801}, "gstreamer_push": {"avg_ms": 0.99, "count": 68, "max_ms": 1.152, "p95_ms": 1.114}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.364, "count": 68, "max_ms": 70.833, "p95_ms": 65.374}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.59, "count": 68, "max_ms": 109.429, "p95_ms": 87.247}, "yolo11_640_fire_11class": {"avg_ms": 54.47, "count": 68, "max_ms": 78.089, "p95_ms": 71.365}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.213, "count": 68, "max_ms": 85.726, "p95_ms": 80.708}}, "truncated_count": 0}, "processing_total": {"avg_ms": 145.354, "count": 68, "max_ms": 190.243, "p95_ms": 170.324}, "read_interval": {"avg_ms": 445.376, "count": 67, "max_ms": 10130.995, "p95_ms": 194.445}, "triton_wait": {"avg_ms": 64.292, "count": 68, "max_ms": 109.441, "p95_ms": 87.26}, "visualize": {"avg_ms": 6.507, "count": 68, "max_ms": 8.405, "p95_ms": 7.929}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.84, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 42, "tracking_summary": 1739}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5314.094, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 34, "h264_decode_error_count": 72, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:04:40+09:00", "last_error_summary": "[h264 @ 0x556c95997ec0] left block unavailable for requested intra mode [h264 @ 0x556c95997ec0] error while decoding MB 0 16, bytestream 3956 [h264 @ 0x556c95b35bc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556c95b35bc0] error while decoding MB 0 21, bytestream 39826 [h264 @ 0x556c94e435c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556c94e435c0] error while decoding MB 0 32, bytestream 40632 [h264 @ 0x556c95997ec0] left block unavailable for requested", "last_exit_at": "2026-07-02T16:04:40+09:00", "last_returncode": 0, "last_stable_duration_sec": 3599.845, "last_started_at": "2026-07-02T15:04:40+09:00", "pid": 11500, "process": {"exists": false, "pid": 11500}, "restart_count": 44, "restart_count_per_hour": 1, "rtsp_timeout_count": 20, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 45}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.417, "sample_count": 2268, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 208709, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:04:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-02T16:04:40+09:00", "last_frame_read_age_sec": 0.169, "last_frame_read_at": "2026-07-02T16:04:40+09:00", "mjpeg": {"age_sec": 0.169, "exists": true, "mtime": "2026-07-02T16:04:40.221601+09:00", "mtime_epoch": 1782975880.2216012, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194068}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:04:40+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1383.0, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 68, "pushed_frames": 68, "read_frames": 67, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.278, "pushed": 2.278, "read": 2.245}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 7.63, "count": 68, "max_ms": 11.219, "p95_ms": 8.801}, "gstreamer_push": {"avg_ms": 0.99, "count": 68, "max_ms": 1.152, "p95_ms": 1.114}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.364, "count": 68, "max_ms": 70.833, "p95_ms": 65.374}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.59, "count": 68, "max_ms": 109.429, "p95_ms": 87.247}, "yolo11_640_fire_11class": {"avg_ms": 54.47, "count": 68, "max_ms": 78.089, "p95_ms": 71.365}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.213, "count": 68, "max_ms": 85.726, "p95_ms": 80.708}}, "truncated_count": 0}, "processing_total": {"avg_ms": 145.354, "count": 68, "max_ms": 190.243, "p95_ms": 170.324}, "read_interval": {"avg_ms": 445.376, "count": 67, "max_ms": 10130.995, "p95_ms": 194.445}, "triton_wait": {"avg_ms": 64.292, "count": 68, "max_ms": 109.441, "p95_ms": 87.26}, "visualize": {"avg_ms": 6.507, "count": 68, "max_ms": 8.405, "p95_ms": 7.929}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.845, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 42, "tracking_summary": 1739}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5421.145, "limit_mb": 8796093022207.996}, "detail": {"pid": 11971, "restart_count": 45, "start_count": 46}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 34, "h264_decode_error_count": 72, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:04:40+09:00", "last_error_summary": "[h264 @ 0x556c95997ec0] left block unavailable for requested intra mode [h264 @ 0x556c95997ec0] error while decoding MB 0 16, bytestream 3956 [h264 @ 0x556c95b35bc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556c95b35bc0] error while decoding MB 0 21, bytestream 39826 [h264 @ 0x556c94e435c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556c94e435c0] error while decoding MB 0 32, bytestream 40632 [h264 @ 0x556c95997ec0] left block unavailable for requested", "last_exit_at": "2026-07-02T16:04:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T16:04:43+09:00", "pid": 11971, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.059, "name": "ffmpeg", "pid": 11971, "rss_mb": 139.059, "state": "S (sleeping)", "threads": 106}, "restart_count": 45, "restart_count_per_hour": 1, "rtsp_timeout_count": 20, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 46}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 152.791, "sample_count": 2220, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 208709, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:04:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.913, "last_frame_pushed_at": "2026-07-02T16:04:40+09:00", "last_frame_read_age_sec": 5.074, "last_frame_read_at": "2026-07-02T16:04:40+09:00", "mjpeg": {"age_sec": 4.902, "exists": true, "mtime": "2026-07-02T16:04:40.393604+09:00", "mtime_epoch": 1782975880.3936036, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191047}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:04:45+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1383.0, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 37, "pushed_frames": 37, "read_frames": 36, "select_timeouts": 18, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.235, "pushed": 1.235, "read": 1.201}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 7.597, "count": 36, "max_ms": 8.935, "p95_ms": 8.638}, "gstreamer_push": {"avg_ms": 0.977, "count": 37, "max_ms": 1.152, "p95_ms": 1.116}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.391, "count": 36, "max_ms": 70.833, "p95_ms": 65.374}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.685, "count": 36, "max_ms": 109.429, "p95_ms": 95.485}, "yolo11_640_fire_11class": {"avg_ms": 53.771, "count": 36, "max_ms": 72.23, "p95_ms": 71.517}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.121, "count": 36, "max_ms": 85.726, "p95_ms": 81.275}}, "truncated_count": 0}, "processing_total": {"avg_ms": 146.938, "count": 37, "max_ms": 190.243, "p95_ms": 174.262}, "read_interval": {"avg_ms": 695.424, "count": 36, "max_ms": 10130.995, "p95_ms": 9530.923}, "triton_wait": {"avg_ms": 64.991, "count": 36, "max_ms": 109.441, "p95_ms": 95.502}, "visualize": {"avg_ms": 6.696, "count": 36, "max_ms": 8.405, "p95_ms": 8.015}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.966, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 42, "tracking_summary": 1745}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5315.496, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T16:04:40+09:00", "last_error_summary": "[h264 @ 0x556c95997ec0] left block unavailable for requested intra mode [h264 @ 0x556c95997ec0] error while decoding MB 0 16, bytestream 3956 [h264 @ 0x556c95b35bc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556c95b35bc0] error while decoding MB 0 21, bytestream 39826 [h264 @ 0x556c94e435c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556c94e435c0] error while decoding MB 0 32, bytestream 40632 [h264 @ 0x556c95997ec0] left block unavailable for requested", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 34, "h264_decode_error_count": 72, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:04:40+09:00", "last_error_summary": "[h264 @ 0x556c95997ec0] left block unavailable for requested intra mode [h264 @ 0x556c95997ec0] error while decoding MB 0 16, bytestream 3956 [h264 @ 0x556c95b35bc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556c95b35bc0] error while decoding MB 0 21, bytestream 39826 [h264 @ 0x556c94e435c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556c94e435c0] error while decoding MB 0 32, bytestream 40632 [h264 @ 0x556c95997ec0] left block unavailable for requested", "last_exit_at": "2026-07-02T16:06:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.887, "last_started_at": "2026-07-02T16:04:43+09:00", "pid": 11971, "process": {"exists": false, "pid": 11971}, "restart_count": 45, "restart_count_per_hour": 1, "rtsp_timeout_count": 20, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 46}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 185.249, "sample_count": 1785, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 209454, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:06:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T16:06:55+09:00", "last_frame_read_age_sec": 0.158, "last_frame_read_at": "2026-07-02T16:06:55+09:00", "mjpeg": {"age_sec": 0.147, "exists": true, "mtime": "2026-07-02T16:06:55.035498+09:00", "mtime_epoch": 1782976015.0354981, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198139}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:06:55+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1383.516, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 181, "pushed_frames": 181, "read_frames": 180, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.056, "pushed": 6.056, "read": 6.023}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.284, "count": 181, "max_ms": 12.747, "p95_ms": 10.028}, "gstreamer_push": {"avg_ms": 0.945, "count": 181, "max_ms": 1.211, "p95_ms": 1.08}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.747, "count": 181, "max_ms": 76.484, "p95_ms": 64.791}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.817, "count": 181, "max_ms": 108.626, "p95_ms": 82.968}, "yolo11_640_fire_11class": {"avg_ms": 50.21, "count": 181, "max_ms": 76.197, "p95_ms": 67.203}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.136, "count": 181, "max_ms": 92.102, "p95_ms": 81.578}}, "truncated_count": 0}, "processing_total": {"avg_ms": 155.582, "count": 181, "max_ms": 204.553, "p95_ms": 184.158}, "read_interval": {"avg_ms": 166.037, "count": 180, "max_ms": 214.775, "p95_ms": 194.52}, "triton_wait": {"avg_ms": 62.132, "count": 181, "max_ms": 108.635, "p95_ms": 86.603}, "visualize": {"avg_ms": 7.141, "count": 181, "max_ms": 9.996, "p95_ms": 8.327}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.887, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 42, "tracking_summary": 1745}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5315.609, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 34, "h264_decode_error_count": 72, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:04:40+09:00", "last_error_summary": "[h264 @ 0x556c95997ec0] left block unavailable for requested intra mode [h264 @ 0x556c95997ec0] error while decoding MB 0 16, bytestream 3956 [h264 @ 0x556c95b35bc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556c95b35bc0] error while decoding MB 0 21, bytestream 39826 [h264 @ 0x556c94e435c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556c94e435c0] error while decoding MB 0 32, bytestream 40632 [h264 @ 0x556c95997ec0] left block unavailable for requested", "last_exit_at": "2026-07-02T16:06:55+09:00", "last_returncode": 0, "last_stable_duration_sec": 131.891, "last_started_at": "2026-07-02T16:04:43+09:00", "pid": 11971, "process": {"exists": false, "pid": 11971}, "restart_count": 45, "restart_count_per_hour": 1, "rtsp_timeout_count": 20, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 46}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 185.249, "sample_count": 1785, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 209454, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:06:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-02T16:06:55+09:00", "last_frame_read_age_sec": 0.162, "last_frame_read_at": "2026-07-02T16:06:55+09:00", "mjpeg": {"age_sec": 0.151, "exists": true, "mtime": "2026-07-02T16:06:55.035498+09:00", "mtime_epoch": 1782976015.0354981, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198139}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:06:55+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1383.516, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 181, "pushed_frames": 181, "read_frames": 180, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.055, "pushed": 6.055, "read": 6.022}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.284, "count": 181, "max_ms": 12.747, "p95_ms": 10.028}, "gstreamer_push": {"avg_ms": 0.945, "count": 181, "max_ms": 1.211, "p95_ms": 1.08}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.747, "count": 181, "max_ms": 76.484, "p95_ms": 64.791}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.817, "count": 181, "max_ms": 108.626, "p95_ms": 82.968}, "yolo11_640_fire_11class": {"avg_ms": 50.21, "count": 181, "max_ms": 76.197, "p95_ms": 67.203}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.136, "count": 181, "max_ms": 92.102, "p95_ms": 81.578}}, "truncated_count": 0}, "processing_total": {"avg_ms": 155.582, "count": 181, "max_ms": 204.553, "p95_ms": 184.158}, "read_interval": {"avg_ms": 166.037, "count": 180, "max_ms": 214.775, "p95_ms": 194.52}, "triton_wait": {"avg_ms": 62.132, "count": 181, "max_ms": 108.635, "p95_ms": 86.603}, "visualize": {"avg_ms": 7.141, "count": 181, "max_ms": 9.996, "p95_ms": 8.327}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.891, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 42, "tracking_summary": 1745}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5421.219, "limit_mb": 8796093022207.996}, "detail": {"pid": 12078, "restart_count": 46, "start_count": 47}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 34, "h264_decode_error_count": 72, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:04:40+09:00", "last_error_summary": "[h264 @ 0x556c95997ec0] left block unavailable for requested intra mode [h264 @ 0x556c95997ec0] error while decoding MB 0 16, bytestream 3956 [h264 @ 0x556c95b35bc0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556c95b35bc0] error while decoding MB 0 21, bytestream 39826 [h264 @ 0x556c94e435c0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x556c94e435c0] error while decoding MB 0 32, bytestream 40632 [h264 @ 0x556c95997ec0] left block unavailable for requested", "last_exit_at": "2026-07-02T16:06:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T16:06:57+09:00", "pid": 12078, "process": {"exists": true, "fd_count": 5, "hwm_mb": 135.926, "name": "ffmpeg", "pid": 12078, "rss_mb": 135.926, "state": "S (sleeping)", "threads": 106}, "restart_count": 46, "restart_count_per_hour": 2, "rtsp_timeout_count": 20, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 47}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 185.32, "sample_count": 1749, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 209454, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:06:55+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.813, "last_frame_pushed_at": "2026-07-02T16:06:55+09:00", "last_frame_read_age_sec": 4.967, "last_frame_read_at": "2026-07-02T16:06:55+09:00", "mjpeg": {"age_sec": 4.796, "exists": true, "mtime": "2026-07-02T16:06:55.195500+09:00", "mtime_epoch": 1782976015.1955004, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 198664}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:06:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1383.516, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 154, "pushed_frames": 154, "read_frames": 153, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.138, "pushed": 5.138, "read": 5.104}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.241, "count": 153, "max_ms": 11.048, "p95_ms": 9.73}, "gstreamer_push": {"avg_ms": 0.951, "count": 154, "max_ms": 1.211, "p95_ms": 1.087}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.906, "count": 153, "max_ms": 76.484, "p95_ms": 64.791}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.631, "count": 153, "max_ms": 104.432, "p95_ms": 82.968}, "yolo11_640_fire_11class": {"avg_ms": 50.486, "count": 153, "max_ms": 76.197, "p95_ms": 67.337}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.937, "count": 153, "max_ms": 92.102, "p95_ms": 81.325}}, "truncated_count": 0}, "processing_total": {"avg_ms": 153.974, "count": 154, "max_ms": 204.553, "p95_ms": 183.994}, "read_interval": {"avg_ms": 164.426, "count": 153, "max_ms": 214.775, "p95_ms": 194.319}, "triton_wait": {"avg_ms": 61.708, "count": 153, "max_ms": 104.444, "p95_ms": 87.631}, "visualize": {"avg_ms": 7.063, "count": 153, "max_ms": 9.171, "p95_ms": 8.176}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.974, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 42, "tracking_summary": 1772}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5314.902, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55e3d0e54d40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e3d0e54d40] error while decoding MB 0 36, bytestream 13411 [h264 @ 0x55e3d052f380] left block unavailable for requested intra mode [h264 @ 0x55e3d052f380] error while decoding MB 0 31, bytestream 2623 [h264 @ 0x55e3d052f380] error while decoding MB 59 21, bytestream -9 [in#0/rtsp @ 0x55e3d03d9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e3d03d9e00] Error retrieving a packet from demuxer", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 35, "h264_decode_error_count": 75, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:12:48+09:00", "last_error_summary": "[h264 @ 0x55e3d0e54d40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e3d0e54d40] error while decoding MB 0 36, bytestream 13411 [h264 @ 0x55e3d052f380] left block unavailable for requested intra mode [h264 @ 0x55e3d052f380] error while decoding MB 0 31, bytestream 2623 [h264 @ 0x55e3d052f380] error while decoding MB 59 21, bytestream -9 [in#0/rtsp @ 0x55e3d03d9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e3d03d9e00] Error retrieving a packet from demuxer", "last_exit_at": "2026-07-02T16:06:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 350.965, "last_started_at": "2026-07-02T16:06:57+09:00", "pid": 12078, "process": {"exists": false, "pid": 12078}, "restart_count": 46, "restart_count_per_hour": 2, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 47}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 130.444, "sample_count": 2811, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 212658, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:12:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.067, "last_frame_pushed_at": "2026-07-02T16:12:48+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-02T16:12:48+09:00", "mjpeg": {"age_sec": 0.051, "exists": true, "mtime": "2026-07-02T16:12:48.904692+09:00", "mtime_epoch": 1782976368.9046922, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196083}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:12:48+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1385.062, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 111, "pushed_frames": 111, "read_frames": 111, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.708, "pushed": 3.708, "read": 3.708}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.059, "count": 111, "max_ms": 6.368, "p95_ms": 5.862}, "gstreamer_push": {"avg_ms": 1.366, "count": 111, "max_ms": 2.409, "p95_ms": 1.647}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.941, "count": 111, "max_ms": 80.802, "p95_ms": 70.01}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.984, "count": 111, "max_ms": 97.999, "p95_ms": 85.189}, "yolo11_640_fire_11class": {"avg_ms": 56.428, "count": 111, "max_ms": 83.059, "p95_ms": 72.271}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.385, "count": 111, "max_ms": 83.294, "p95_ms": 78.701}}, "truncated_count": 0}, "processing_total": {"avg_ms": 78.641, "count": 111, "max_ms": 115.403, "p95_ms": 98.019}, "read_interval": {"avg_ms": 269.854, "count": 111, "max_ms": 19395.996, "p95_ms": 118.668}, "triton_wait": {"avg_ms": 65.791, "count": 111, "max_ms": 98.014, "p95_ms": 85.202}, "visualize": {"avg_ms": 3.749, "count": 111, "max_ms": 5.44, "p95_ms": 4.804}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.938, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 42, "tracking_summary": 1772}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5314.609, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T16:12:48+09:00", "last_error_summary": "[h264 @ 0x55e3d0e54d40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e3d0e54d40] error while decoding MB 0 36, bytestream 13411 [h264 @ 0x55e3d052f380] left block unavailable for requested intra mode [h264 @ 0x55e3d052f380] error while decoding MB 0 31, bytestream 2623 [h264 @ 0x55e3d052f380] error while decoding MB 59 21, bytestream -9 [in#0/rtsp @ 0x55e3d03d9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e3d03d9e00] Error retrieving a packet from demuxer", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 35, "h264_decode_error_count": 75, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:12:48+09:00", "last_error_summary": "[h264 @ 0x55e3d0e54d40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e3d0e54d40] error while decoding MB 0 36, bytestream 13411 [h264 @ 0x55e3d052f380] left block unavailable for requested intra mode [h264 @ 0x55e3d052f380] error while decoding MB 0 31, bytestream 2623 [h264 @ 0x55e3d052f380] error while decoding MB 59 21, bytestream -9 [in#0/rtsp @ 0x55e3d03d9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e3d03d9e00] Error retrieving a packet from demuxer", "last_exit_at": "2026-07-02T16:12:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 350.993, "last_started_at": "2026-07-02T16:06:57+09:00", "pid": 12078, "process": {"exists": false, "pid": 12078}, "restart_count": 46, "restart_count_per_hour": 2, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 47}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 130.379, "sample_count": 2812, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 212659, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:12:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-02T16:12:48+09:00", "last_frame_read_age_sec": 0.082, "last_frame_read_at": "2026-07-02T16:12:48+09:00", "mjpeg": {"age_sec": 0.079, "exists": true, "mtime": "2026-07-02T16:12:48.904692+09:00", "mtime_epoch": 1782976368.9046922, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196083}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:12:48+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1385.062, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 112, "pushed_frames": 112, "read_frames": 111, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.738, "pushed": 3.738, "read": 3.704}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.058, "count": 112, "max_ms": 6.368, "p95_ms": 5.862}, "gstreamer_push": {"avg_ms": 1.365, "count": 112, "max_ms": 2.409, "p95_ms": 1.647}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.866, "count": 112, "max_ms": 80.802, "p95_ms": 70.01}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.932, "count": 112, "max_ms": 97.999, "p95_ms": 85.189}, "yolo11_640_fire_11class": {"avg_ms": 56.341, "count": 112, "max_ms": 83.059, "p95_ms": 72.271}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.285, "count": 112, "max_ms": 83.294, "p95_ms": 78.701}}, "truncated_count": 0}, "processing_total": {"avg_ms": 78.581, "count": 112, "max_ms": 115.403, "p95_ms": 98.019}, "read_interval": {"avg_ms": 269.854, "count": 111, "max_ms": 19395.996, "p95_ms": 118.668}, "triton_wait": {"avg_ms": 65.723, "count": 112, "max_ms": 98.014, "p95_ms": 85.202}, "visualize": {"avg_ms": 3.757, "count": 112, "max_ms": 5.44, "p95_ms": 4.804}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.966, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 42, "tracking_summary": 1772}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5314.434, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 35, "h264_decode_error_count": 75, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:12:48+09:00", "last_error_summary": "[h264 @ 0x55e3d0e54d40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e3d0e54d40] error while decoding MB 0 36, bytestream 13411 [h264 @ 0x55e3d052f380] left block unavailable for requested intra mode [h264 @ 0x55e3d052f380] error while decoding MB 0 31, bytestream 2623 [h264 @ 0x55e3d052f380] error while decoding MB 59 21, bytestream -9 [in#0/rtsp @ 0x55e3d03d9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e3d03d9e00] Error retrieving a packet from demuxer", "last_exit_at": "2026-07-02T16:12:48+09:00", "last_returncode": 0, "last_stable_duration_sec": 351.002, "last_started_at": "2026-07-02T16:06:57+09:00", "pid": 12078, "process": {"exists": false, "pid": 12078}, "restart_count": 46, "restart_count_per_hour": 2, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 47}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 130.379, "sample_count": 2812, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 212659, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:12:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.017, "last_frame_pushed_at": "2026-07-02T16:12:48+09:00", "last_frame_read_age_sec": 0.091, "last_frame_read_at": "2026-07-02T16:12:48+09:00", "mjpeg": {"age_sec": 0.087, "exists": true, "mtime": "2026-07-02T16:12:48.904692+09:00", "mtime_epoch": 1782976368.9046922, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196083}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:12:48+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1385.062, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 112, "pushed_frames": 112, "read_frames": 111, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.737, "pushed": 3.737, "read": 3.703}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.058, "count": 112, "max_ms": 6.368, "p95_ms": 5.862}, "gstreamer_push": {"avg_ms": 1.365, "count": 112, "max_ms": 2.409, "p95_ms": 1.647}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.866, "count": 112, "max_ms": 80.802, "p95_ms": 70.01}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.932, "count": 112, "max_ms": 97.999, "p95_ms": 85.189}, "yolo11_640_fire_11class": {"avg_ms": 56.341, "count": 112, "max_ms": 83.059, "p95_ms": 72.271}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.285, "count": 112, "max_ms": 83.294, "p95_ms": 78.701}}, "truncated_count": 0}, "processing_total": {"avg_ms": 78.581, "count": 112, "max_ms": 115.403, "p95_ms": 98.019}, "read_interval": {"avg_ms": 269.854, "count": 111, "max_ms": 19395.996, "p95_ms": 118.668}, "triton_wait": {"avg_ms": 65.723, "count": 112, "max_ms": 98.014, "p95_ms": 85.202}, "visualize": {"avg_ms": 3.757, "count": 112, "max_ms": 5.44, "p95_ms": 4.804}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.974, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 42, "tracking_summary": 1772}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5420.66, "limit_mb": 8796093022207.996}, "detail": {"pid": 12185, "restart_count": 47, "start_count": 48}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 35, "h264_decode_error_count": 75, "h264_decode_error_count_per_hour": 7, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:12:48+09:00", "last_error_summary": "[h264 @ 0x55e3d0e54d40] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55e3d0e54d40] error while decoding MB 0 36, bytestream 13411 [h264 @ 0x55e3d052f380] left block unavailable for requested intra mode [h264 @ 0x55e3d052f380] error while decoding MB 0 31, bytestream 2623 [h264 @ 0x55e3d052f380] error while decoding MB 59 21, bytestream -9 [in#0/rtsp @ 0x55e3d03d9e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55e3d03d9e00] Error retrieving a packet from demuxer", "last_exit_at": "2026-07-02T16:12:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T16:12:51+09:00", "pid": 12185, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.531, "name": "ffmpeg", "pid": 12185, "rss_mb": 136.531, "state": "S (sleeping)", "threads": 106}, "restart_count": 47, "restart_count_per_hour": 3, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 48}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 130.618, "sample_count": 2768, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 212659, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:12:48+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.237, "last_frame_pushed_at": "2026-07-02T16:12:48+09:00", "last_frame_read_age_sec": 4.311, "last_frame_read_at": "2026-07-02T16:12:48+09:00", "mjpeg": {"age_sec": 4.224, "exists": true, "mtime": "2026-07-02T16:12:48.988693+09:00", "mtime_epoch": 1782976368.9886935, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 194359}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:12:53+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1385.062, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 68, "pushed_frames": 68, "read_frames": 67, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.268, "pushed": 2.268, "read": 2.234}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 3.917, "count": 68, "max_ms": 4.787, "p95_ms": 4.42}, "gstreamer_push": {"avg_ms": 1.335, "count": 68, "max_ms": 1.947, "p95_ms": 1.482}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 56.349, "count": 68, "max_ms": 78.675, "p95_ms": 69.031}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.842, "count": 68, "max_ms": 87.419, "p95_ms": 82.007}, "yolo11_640_fire_11class": {"avg_ms": 56.914, "count": 68, "max_ms": 77.13, "p95_ms": 71.535}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.62, "count": 68, "max_ms": 83.294, "p95_ms": 78.755}}, "truncated_count": 0}, "processing_total": {"avg_ms": 78.944, "count": 68, "max_ms": 100.264, "p95_ms": 95.474}, "read_interval": {"avg_ms": 384.212, "count": 67, "max_ms": 19395.996, "p95_ms": 112.18}, "triton_wait": {"avg_ms": 66.353, "count": 68, "max_ms": 87.433, "p95_ms": 82.104}, "visualize": {"avg_ms": 3.681, "count": 68, "max_ms": 4.804, "p95_ms": 4.22}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.985, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 42, "tracking_summary": 1804}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5326.809, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5597e69f02c0] error while decoding MB 61 14, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 36, "h264_decode_error_count": 76, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:20:15+09:00", "last_error_summary": "[h264 @ 0x5597e69f02c0] error while decoding MB 61 14, bytestream -5", "last_exit_at": "2026-07-02T16:12:51+09:00", "last_returncode": 0, "last_stable_duration_sec": 444.438, "last_started_at": "2026-07-02T16:12:51+09:00", "pid": 12185, "process": {"exists": false, "pid": 12185}, "restart_count": 47, "restart_count_per_hour": 3, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 48}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 141.103, "sample_count": 2539, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 216557, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:20:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.075, "last_frame_pushed_at": "2026-07-02T16:20:15+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-02T16:20:15+09:00", "mjpeg": {"age_sec": 0.058, "exists": true, "mtime": "2026-07-02T16:20:15.591498+09:00", "mtime_epoch": 1782976815.5914977, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196329}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:20:15+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1386.867, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 223, "pushed_frames": 223, "read_frames": 223, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.456, "pushed": 7.456, "read": 7.456}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.684, "count": 223, "max_ms": 10.106, "p95_ms": 7.847}, "gstreamer_push": {"avg_ms": 1.111, "count": 223, "max_ms": 1.456, "p95_ms": 1.229}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.264, "count": 223, "max_ms": 75.774, "p95_ms": 68.657}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.004, "count": 223, "max_ms": 101.682, "p95_ms": 82.607}, "yolo11_640_fire_11class": {"avg_ms": 53.321, "count": 223, "max_ms": 78.3, "p95_ms": 70.363}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.371, "count": 223, "max_ms": 93.633, "p95_ms": 84.969}}, "truncated_count": 0}, "processing_total": {"avg_ms": 121.442, "count": 223, "max_ms": 160.663, "p95_ms": 145.391}, "read_interval": {"avg_ms": 134.419, "count": 223, "max_ms": 173.001, "p95_ms": 158.246}, "triton_wait": {"avg_ms": 66.112, "count": 223, "max_ms": 101.694, "p95_ms": 86.612}, "visualize": {"avg_ms": 5.353, "count": 223, "max_ms": 8.143, "p95_ms": 6.674}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.908, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 42, "tracking_summary": 1804}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5319.441, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T16:20:15+09:00", "last_error_summary": "[h264 @ 0x5597e69f02c0] error while decoding MB 61 14, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 36, "h264_decode_error_count": 76, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:20:15+09:00", "last_error_summary": "[h264 @ 0x5597e69f02c0] error while decoding MB 61 14, bytestream -5", "last_exit_at": "2026-07-02T16:20:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 444.499, "last_started_at": "2026-07-02T16:12:51+09:00", "pid": 12185, "process": {"exists": false, "pid": 12185}, "restart_count": 47, "restart_count_per_hour": 3, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 48}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 141.103, "sample_count": 2540, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 216558, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:20:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T16:20:15+09:00", "last_frame_read_age_sec": 0.126, "last_frame_read_at": "2026-07-02T16:20:15+09:00", "mjpeg": {"age_sec": 0.119, "exists": true, "mtime": "2026-07-02T16:20:15.591498+09:00", "mtime_epoch": 1782976815.5914977, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196329}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:20:15+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1386.867, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 224, "pushed_frames": 224, "read_frames": 223, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.474, "pushed": 7.474, "read": 7.441}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.688, "count": 223, "max_ms": 10.106, "p95_ms": 7.847}, "gstreamer_push": {"avg_ms": 1.111, "count": 224, "max_ms": 1.456, "p95_ms": 1.229}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.164, "count": 223, "max_ms": 75.774, "p95_ms": 68.657}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.998, "count": 223, "max_ms": 101.682, "p95_ms": 82.607}, "yolo11_640_fire_11class": {"avg_ms": 53.209, "count": 223, "max_ms": 78.3, "p95_ms": 70.363}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.216, "count": 223, "max_ms": 93.633, "p95_ms": 84.969}}, "truncated_count": 0}, "processing_total": {"avg_ms": 121.427, "count": 224, "max_ms": 160.663, "p95_ms": 145.391}, "read_interval": {"avg_ms": 134.419, "count": 223, "max_ms": 173.001, "p95_ms": 158.246}, "triton_wait": {"avg_ms": 65.991, "count": 223, "max_ms": 101.694, "p95_ms": 86.612}, "visualize": {"avg_ms": 5.371, "count": 223, "max_ms": 9.036, "p95_ms": 6.715}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.969, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 42, "tracking_summary": 1804}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5319.324, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 36, "h264_decode_error_count": 76, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:20:15+09:00", "last_error_summary": "[h264 @ 0x5597e69f02c0] error while decoding MB 61 14, bytestream -5", "last_exit_at": "2026-07-02T16:20:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 444.506, "last_started_at": "2026-07-02T16:12:51+09:00", "pid": 12185, "process": {"exists": false, "pid": 12185}, "restart_count": 47, "restart_count_per_hour": 3, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 48}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 141.103, "sample_count": 2540, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 216558, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:20:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-02T16:20:15+09:00", "last_frame_read_age_sec": 0.132, "last_frame_read_at": "2026-07-02T16:20:15+09:00", "mjpeg": {"age_sec": 0.126, "exists": true, "mtime": "2026-07-02T16:20:15.591498+09:00", "mtime_epoch": 1782976815.5914977, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196329}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:20:15+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1386.867, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 224, "pushed_frames": 224, "read_frames": 223, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.473, "pushed": 7.473, "read": 7.439}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.688, "count": 223, "max_ms": 10.106, "p95_ms": 7.847}, "gstreamer_push": {"avg_ms": 1.111, "count": 224, "max_ms": 1.456, "p95_ms": 1.229}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.164, "count": 223, "max_ms": 75.774, "p95_ms": 68.657}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.998, "count": 223, "max_ms": 101.682, "p95_ms": 82.607}, "yolo11_640_fire_11class": {"avg_ms": 53.209, "count": 223, "max_ms": 78.3, "p95_ms": 70.363}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.216, "count": 223, "max_ms": 93.633, "p95_ms": 84.969}}, "truncated_count": 0}, "processing_total": {"avg_ms": 121.427, "count": 224, "max_ms": 160.663, "p95_ms": 145.391}, "read_interval": {"avg_ms": 134.419, "count": 223, "max_ms": 173.001, "p95_ms": 158.246}, "triton_wait": {"avg_ms": 65.991, "count": 223, "max_ms": 101.694, "p95_ms": 86.612}, "visualize": {"avg_ms": 5.371, "count": 223, "max_ms": 9.036, "p95_ms": 6.715}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.975, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 42, "tracking_summary": 1804}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5427.441, "limit_mb": 8796093022207.996}, "detail": {"pid": 12292, "restart_count": 48, "start_count": 49}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 36, "h264_decode_error_count": 76, "h264_decode_error_count_per_hour": 8, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:20:15+09:00", "last_error_summary": "[h264 @ 0x5597e69f02c0] error while decoding MB 61 14, bytestream -5", "last_exit_at": "2026-07-02T16:20:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T16:20:17+09:00", "pid": 12292, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.184, "name": "ffmpeg", "pid": 12292, "rss_mb": 141.184, "state": "S (sleeping)", "threads": 106}, "restart_count": 48, "restart_count_per_hour": 4, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 49}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 141.172, "sample_count": 2508, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 216558, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:20:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.084, "last_frame_pushed_at": "2026-07-02T16:20:15+09:00", "last_frame_read_age_sec": 4.203, "last_frame_read_at": "2026-07-02T16:20:15+09:00", "mjpeg": {"age_sec": 4.065, "exists": true, "mtime": "2026-07-02T16:20:15.723500+09:00", "mtime_epoch": 1782976815.7234998, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 189559}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:20:19+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1386.867, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 192, "pushed_frames": 192, "read_frames": 191, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.421, "pushed": 6.421, "read": 6.388}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.735, "count": 192, "max_ms": 10.106, "p95_ms": 7.966}, "gstreamer_push": {"avg_ms": 1.114, "count": 192, "max_ms": 1.456, "p95_ms": 1.233}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.402, "count": 192, "max_ms": 75.774, "p95_ms": 68.711}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.238, "count": 192, "max_ms": 101.682, "p95_ms": 83.537}, "yolo11_640_fire_11class": {"avg_ms": 53.469, "count": 192, "max_ms": 78.3, "p95_ms": 70.402}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.482, "count": 192, "max_ms": 93.633, "p95_ms": 86.108}}, "truncated_count": 0}, "processing_total": {"avg_ms": 122.264, "count": 192, "max_ms": 160.663, "p95_ms": 147.089}, "read_interval": {"avg_ms": 135.272, "count": 191, "max_ms": 173.001, "p95_ms": 160.422}, "triton_wait": {"avg_ms": 66.283, "count": 192, "max_ms": 101.694, "p95_ms": 86.819}, "visualize": {"avg_ms": 5.411, "count": 192, "max_ms": 9.036, "p95_ms": 6.747}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.9, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 35, "tracking_summary": 1828}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5278.766, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55bcf5062280] cabac decode of qscale diff failed at 5 49 [h264 @ 0x55bcf5062280] error while decoding MB 5 49, bytestream -4 [in#0/rtsp @ 0x55bcf5001e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55bcf5001e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 31, "h264_decode_error_count": 32, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:27:20+09:00", "last_error_summary": "[h264 @ 0x55bcf5062280] cabac decode of qscale diff failed at 5 49 [h264 @ 0x55bcf5062280] error while decoding MB 5 49, bytestream -4 [in#0/rtsp @ 0x55bcf5001e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55bcf5001e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T15:45:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 2497.256, "last_started_at": "2026-07-02T15:45:43+09:00", "pid": 11759, "process": {"exists": false, "pid": 11759}, "restart_count": 30, "restart_count_per_hour": 1, "rtsp_timeout_count": 44, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 31}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 151.127, "sample_count": 2561, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 219451, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:27:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.076, "last_frame_pushed_at": "2026-07-02T16:27:20+09:00", "last_frame_read_age_sec": 0.065, "last_frame_read_at": "2026-07-02T16:27:20+09:00", "mjpeg": {"age_sec": 0.059, "exists": true, "mtime": "2026-07-02T16:27:20.522089+09:00", "mtime_epoch": 1782977240.522089, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 265157}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T16:27:20+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1133.418, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.739, "pushed": 0.739, "read": 0.796}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.005, "count": 14, "max_ms": 6.951, "p95_ms": 6.951}, "gstreamer_push": {"avg_ms": 1.307, "count": 13, "max_ms": 1.544, "p95_ms": 1.544}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.768, "count": 14, "max_ms": 72.078, "p95_ms": 72.078}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.579, "count": 14, "max_ms": 78.876, "p95_ms": 78.876}, "yolo11_640_fire_11class": {"avg_ms": 54.845, "count": 14, "max_ms": 73.311, "p95_ms": 73.311}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.789, "count": 14, "max_ms": 86.457, "p95_ms": 86.457}}, "truncated_count": 0}, "processing_total": {"avg_ms": 74.838, "count": 13, "max_ms": 99.735, "p95_ms": 99.735}, "read_interval": {"avg_ms": 2418.716, "count": 14, "max_ms": 16462.393, "p95_ms": 16462.393}, "triton_wait": {"avg_ms": 59.731, "count": 14, "max_ms": 86.468, "p95_ms": 86.468}, "visualize": {"avg_ms": 3.649, "count": 14, "max_ms": 4.615, "p95_ms": 4.615}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.586, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 35, "tracking_summary": 1828}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5278.496, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T16:27:20+09:00", "last_error_summary": "[h264 @ 0x55bcf5062280] cabac decode of qscale diff failed at 5 49 [h264 @ 0x55bcf5062280] error while decoding MB 5 49, bytestream -4 [in#0/rtsp @ 0x55bcf5001e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55bcf5001e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 31, "h264_decode_error_count": 32, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:27:20+09:00", "last_error_summary": "[h264 @ 0x55bcf5062280] cabac decode of qscale diff failed at 5 49 [h264 @ 0x55bcf5062280] error while decoding MB 5 49, bytestream -4 [in#0/rtsp @ 0x55bcf5001e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55bcf5001e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T16:27:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 2497.275, "last_started_at": "2026-07-02T15:45:43+09:00", "pid": 11759, "process": {"exists": false, "pid": 11759}, "restart_count": 30, "restart_count_per_hour": 1, "rtsp_timeout_count": 44, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 31}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 151.127, "sample_count": 2562, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 219451, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:27:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-02T16:27:20+09:00", "last_frame_read_age_sec": 0.084, "last_frame_read_at": "2026-07-02T16:27:20+09:00", "mjpeg": {"age_sec": 0.079, "exists": true, "mtime": "2026-07-02T16:27:20.522089+09:00", "mtime_epoch": 1782977240.522089, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 265157}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T16:27:20+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1133.418, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.795, "pushed": 0.795, "read": 0.795}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.005, "count": 14, "max_ms": 6.951, "p95_ms": 6.951}, "gstreamer_push": {"avg_ms": 1.301, "count": 14, "max_ms": 1.544, "p95_ms": 1.544}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.768, "count": 14, "max_ms": 72.078, "p95_ms": 72.078}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.579, "count": 14, "max_ms": 78.876, "p95_ms": 78.876}, "yolo11_640_fire_11class": {"avg_ms": 54.845, "count": 14, "max_ms": 73.311, "p95_ms": 73.311}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.789, "count": 14, "max_ms": 86.457, "p95_ms": 86.457}}, "truncated_count": 0}, "processing_total": {"avg_ms": 74.728, "count": 14, "max_ms": 99.735, "p95_ms": 99.735}, "read_interval": {"avg_ms": 2418.716, "count": 14, "max_ms": 16462.393, "p95_ms": 16462.393}, "triton_wait": {"avg_ms": 59.731, "count": 14, "max_ms": 86.468, "p95_ms": 86.468}, "visualize": {"avg_ms": 3.649, "count": 14, "max_ms": 4.615, "p95_ms": 4.615}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.605, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 35, "tracking_summary": 1828}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5277.742, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 31, "h264_decode_error_count": 32, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:27:20+09:00", "last_error_summary": "[h264 @ 0x55bcf5062280] cabac decode of qscale diff failed at 5 49 [h264 @ 0x55bcf5062280] error while decoding MB 5 49, bytestream -4 [in#0/rtsp @ 0x55bcf5001e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55bcf5001e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T16:27:20+09:00", "last_returncode": 0, "last_stable_duration_sec": 2497.283, "last_started_at": "2026-07-02T15:45:43+09:00", "pid": 11759, "process": {"exists": false, "pid": 11759}, "restart_count": 30, "restart_count_per_hour": 1, "rtsp_timeout_count": 44, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 31}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 151.127, "sample_count": 2562, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 219451, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:27:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.017, "last_frame_pushed_at": "2026-07-02T16:27:20+09:00", "last_frame_read_age_sec": 0.092, "last_frame_read_at": "2026-07-02T16:27:20+09:00", "mjpeg": {"age_sec": 0.086, "exists": true, "mtime": "2026-07-02T16:27:20.522089+09:00", "mtime_epoch": 1782977240.522089, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 265157}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T16:27:20+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1133.418, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.795, "pushed": 0.795, "read": 0.795}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.005, "count": 14, "max_ms": 6.951, "p95_ms": 6.951}, "gstreamer_push": {"avg_ms": 1.301, "count": 14, "max_ms": 1.544, "p95_ms": 1.544}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.768, "count": 14, "max_ms": 72.078, "p95_ms": 72.078}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.579, "count": 14, "max_ms": 78.876, "p95_ms": 78.876}, "yolo11_640_fire_11class": {"avg_ms": 54.845, "count": 14, "max_ms": 73.311, "p95_ms": 73.311}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.789, "count": 14, "max_ms": 86.457, "p95_ms": 86.457}}, "truncated_count": 0}, "processing_total": {"avg_ms": 74.728, "count": 14, "max_ms": 99.735, "p95_ms": 99.735}, "read_interval": {"avg_ms": 2418.716, "count": 14, "max_ms": 16462.393, "p95_ms": 16462.393}, "triton_wait": {"avg_ms": 59.731, "count": 14, "max_ms": 86.468, "p95_ms": 86.468}, "visualize": {"avg_ms": 3.649, "count": 14, "max_ms": 4.615, "p95_ms": 4.615}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 17.613, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 35, "tracking_summary": 1828}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5449.656, "limit_mb": 8796093022207.996}, "detail": {"pid": 12459, "restart_count": 31, "start_count": 32}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 31, "h264_decode_error_count": 32, "h264_decode_error_count_per_hour": 3, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:27:20+09:00", "last_error_summary": "[h264 @ 0x55bcf5062280] cabac decode of qscale diff failed at 5 49 [h264 @ 0x55bcf5062280] error while decoding MB 5 49, bytestream -4 [in#0/rtsp @ 0x55bcf5001e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55bcf5001e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T16:27:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T16:27:22+09:00", "pid": 12459, "process": {"exists": true, "fd_count": 5, "hwm_mb": 207.324, "name": "ffmpeg", "pid": 12459, "rss_mb": 207.324, "state": "S (sleeping)", "threads": 106}, "restart_count": 31, "restart_count_per_hour": 2, "rtsp_timeout_count": 44, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 32}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 151.451, "sample_count": 2519, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 219451, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:27:20+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 4.259, "last_frame_pushed_at": "2026-07-02T16:27:20+09:00", "last_frame_read_age_sec": 4.334, "last_frame_read_at": "2026-07-02T16:27:20+09:00", "mjpeg": {"age_sec": 4.241, "exists": true, "mtime": "2026-07-02T16:27:20.610090+09:00", "mtime_epoch": 1782977240.6100903, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 263025}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T16:27:24+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1376.699, "name": "celery", "pid": 267, "rss_mb": 1133.418, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.641, "pushed": 0.641, "read": 0.641}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.005, "count": 14, "max_ms": 6.951, "p95_ms": 6.951}, "gstreamer_push": {"avg_ms": 1.301, "count": 14, "max_ms": 1.544, "p95_ms": 1.544}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.768, "count": 14, "max_ms": 72.078, "p95_ms": 72.078}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 56.579, "count": 14, "max_ms": 78.876, "p95_ms": 78.876}, "yolo11_640_fire_11class": {"avg_ms": 54.845, "count": 14, "max_ms": 73.311, "p95_ms": 73.311}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 54.789, "count": 14, "max_ms": 86.457, "p95_ms": 86.457}}, "truncated_count": 0}, "processing_total": {"avg_ms": 74.728, "count": 14, "max_ms": 99.735, "p95_ms": 99.735}, "read_interval": {"avg_ms": 2418.716, "count": 14, "max_ms": 16462.393, "p95_ms": 16462.393}, "triton_wait": {"avg_ms": 59.731, "count": 14, "max_ms": 86.468, "p95_ms": 86.468}, "visualize": {"avg_ms": 3.649, "count": 14, "max_ms": 4.615, "p95_ms": 4.615}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 21.855, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 42, "tracking_summary": 1867}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5353.551, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55fe722619c0] error while decoding MB 23 16, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 37, "h264_decode_error_count": 77, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:33:42+09:00", "last_error_summary": "[h264 @ 0x55fe722619c0] error while decoding MB 23 16, bytestream -7", "last_exit_at": "2026-07-02T16:20:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 804.869, "last_started_at": "2026-07-02T16:20:17+09:00", "pid": 12292, "process": {"exists": false, "pid": 12292}, "restart_count": 48, "restart_count_per_hour": 4, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 49}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 165.987, "sample_count": 2455, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 224042, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:33:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.049, "last_frame_pushed_at": "2026-07-02T16:33:42+09:00", "last_frame_read_age_sec": 0.037, "last_frame_read_at": "2026-07-02T16:33:42+09:00", "mjpeg": {"age_sec": 0.031, "exists": true, "mtime": "2026-07-02T16:33:42.625037+09:00", "mtime_epoch": 1782977622.6250367, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181343}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:33:42+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1390.219, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 188, "pushed_frames": 188, "read_frames": 188, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.271, "pushed": 6.271, "read": 6.271}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.039, "count": 187, "max_ms": 13.157, "p95_ms": 10.261}, "gstreamer_push": {"avg_ms": 1.009, "count": 188, "max_ms": 1.782, "p95_ms": 1.171}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.695, "count": 187, "max_ms": 73.68, "p95_ms": 63.771}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.309, "count": 187, "max_ms": 104.408, "p95_ms": 84.747}, "yolo11_640_fire_11class": {"avg_ms": 50.231, "count": 187, "max_ms": 79.196, "p95_ms": 67.116}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.425, "count": 187, "max_ms": 100.25, "p95_ms": 83.638}}, "truncated_count": 0}, "processing_total": {"avg_ms": 148.515, "count": 188, "max_ms": 203.875, "p95_ms": 182.617}, "read_interval": {"avg_ms": 159.993, "count": 188, "max_ms": 213.536, "p95_ms": 194.096}, "triton_wait": {"avg_ms": 65.042, "count": 187, "max_ms": 104.421, "p95_ms": 89.962}, "visualize": {"avg_ms": 6.588, "count": 187, "max_ms": 11.948, "p95_ms": 8.501}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.977, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 42, "tracking_summary": 1867}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5352.984, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T16:33:42+09:00", "last_error_summary": "[h264 @ 0x55fe722619c0] error while decoding MB 23 16, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 37, "h264_decode_error_count": 77, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:33:42+09:00", "last_error_summary": "[h264 @ 0x55fe722619c0] error while decoding MB 23 16, bytestream -7", "last_exit_at": "2026-07-02T16:33:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 804.971, "last_started_at": "2026-07-02T16:20:17+09:00", "pid": 12292, "process": {"exists": false, "pid": 12292}, "restart_count": 48, "restart_count_per_hour": 4, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 49}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 165.987, "sample_count": 2455, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 224043, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:33:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T16:33:42+09:00", "last_frame_read_age_sec": 0.139, "last_frame_read_at": "2026-07-02T16:33:42+09:00", "mjpeg": {"age_sec": 0.133, "exists": true, "mtime": "2026-07-02T16:33:42.625037+09:00", "mtime_epoch": 1782977622.6250367, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181343}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:33:42+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1390.219, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 188, "pushed_frames": 188, "read_frames": 187, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.281, "pushed": 6.281, "read": 6.248}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.041, "count": 188, "max_ms": 13.157, "p95_ms": 10.261}, "gstreamer_push": {"avg_ms": 1.009, "count": 188, "max_ms": 1.782, "p95_ms": 1.171}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.658, "count": 188, "max_ms": 73.68, "p95_ms": 63.771}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.235, "count": 188, "max_ms": 104.408, "p95_ms": 84.747}, "yolo11_640_fire_11class": {"avg_ms": 50.19, "count": 188, "max_ms": 79.196, "p95_ms": 67.116}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.341, "count": 188, "max_ms": 100.25, "p95_ms": 83.638}}, "truncated_count": 0}, "processing_total": {"avg_ms": 148.482, "count": 188, "max_ms": 203.875, "p95_ms": 182.617}, "read_interval": {"avg_ms": 160.045, "count": 187, "max_ms": 213.536, "p95_ms": 194.096}, "triton_wait": {"avg_ms": 64.954, "count": 188, "max_ms": 104.421, "p95_ms": 89.962}, "visualize": {"avg_ms": 6.586, "count": 188, "max_ms": 11.948, "p95_ms": 8.501}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.929, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 42, "tracking_summary": 1867}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5352.977, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 37, "h264_decode_error_count": 77, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:33:42+09:00", "last_error_summary": "[h264 @ 0x55fe722619c0] error while decoding MB 23 16, bytestream -7", "last_exit_at": "2026-07-02T16:33:42+09:00", "last_returncode": 0, "last_stable_duration_sec": 804.976, "last_started_at": "2026-07-02T16:20:17+09:00", "pid": 12292, "process": {"exists": false, "pid": 12292}, "restart_count": 48, "restart_count_per_hour": 4, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 49}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 165.987, "sample_count": 2455, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 224043, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:33:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.009, "last_frame_pushed_at": "2026-07-02T16:33:42+09:00", "last_frame_read_age_sec": 0.144, "last_frame_read_at": "2026-07-02T16:33:42+09:00", "mjpeg": {"age_sec": 0.138, "exists": true, "mtime": "2026-07-02T16:33:42.625037+09:00", "mtime_epoch": 1782977622.6250367, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181343}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:33:42+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1390.219, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 188, "pushed_frames": 188, "read_frames": 187, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.28, "pushed": 6.28, "read": 6.247}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 8.043, "count": 187, "max_ms": 13.157, "p95_ms": 10.261}, "gstreamer_push": {"avg_ms": 1.009, "count": 188, "max_ms": 1.782, "p95_ms": 1.171}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.67, "count": 187, "max_ms": 73.68, "p95_ms": 63.771}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.281, "count": 187, "max_ms": 104.408, "p95_ms": 84.747}, "yolo11_640_fire_11class": {"avg_ms": 50.209, "count": 187, "max_ms": 79.196, "p95_ms": 67.116}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.36, "count": 187, "max_ms": 100.25, "p95_ms": 83.638}}, "truncated_count": 0}, "processing_total": {"avg_ms": 148.482, "count": 188, "max_ms": 203.875, "p95_ms": 182.617}, "read_interval": {"avg_ms": 160.045, "count": 187, "max_ms": 213.536, "p95_ms": 194.096}, "triton_wait": {"avg_ms": 64.992, "count": 187, "max_ms": 104.421, "p95_ms": 89.962}, "visualize": {"avg_ms": 6.589, "count": 187, "max_ms": 11.948, "p95_ms": 8.501}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.934, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 42, "tracking_summary": 1867}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5464.434, "limit_mb": 8796093022207.996}, "detail": {"pid": 12671, "restart_count": 49, "start_count": 50}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 37, "h264_decode_error_count": 77, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:33:42+09:00", "last_error_summary": "[h264 @ 0x55fe722619c0] error while decoding MB 23 16, bytestream -7", "last_exit_at": "2026-07-02T16:33:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T16:33:44+09:00", "pid": 12671, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.449, "name": "ffmpeg", "pid": 12671, "rss_mb": 139.449, "state": "S (sleeping)", "threads": 106}, "restart_count": 49, "restart_count_per_hour": 5, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 50}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 166.016, "sample_count": 2415, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 224043, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:33:42+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.068, "last_frame_pushed_at": "2026-07-02T16:33:42+09:00", "last_frame_read_age_sec": 4.202, "last_frame_read_at": "2026-07-02T16:33:42+09:00", "mjpeg": {"age_sec": 4.053, "exists": true, "mtime": "2026-07-02T16:33:42.769040+09:00", "mtime_epoch": 1782977622.7690399, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 181634}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:33:46+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1390.477, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 162, "pushed_frames": 162, "read_frames": 161, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.415, "pushed": 5.415, "read": 5.382}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.996, "count": 162, "max_ms": 12.497, "p95_ms": 9.935}, "gstreamer_push": {"avg_ms": 1.007, "count": 162, "max_ms": 1.768, "p95_ms": 1.169}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.415, "count": 162, "max_ms": 73.68, "p95_ms": 63.771}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.984, "count": 162, "max_ms": 104.408, "p95_ms": 84.747}, "yolo11_640_fire_11class": {"avg_ms": 50.052, "count": 162, "max_ms": 79.196, "p95_ms": 67.116}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.126, "count": 162, "max_ms": 100.25, "p95_ms": 83.638}}, "truncated_count": 0}, "processing_total": {"avg_ms": 148.882, "count": 162, "max_ms": 203.875, "p95_ms": 182.617}, "read_interval": {"avg_ms": 160.518, "count": 161, "max_ms": 213.536, "p95_ms": 194.096}, "triton_wait": {"avg_ms": 64.926, "count": 162, "max_ms": 104.421, "p95_ms": 90.268}, "visualize": {"avg_ms": 6.602, "count": 162, "max_ms": 11.948, "p95_ms": 8.502}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.915, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1873}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5353.641, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T16:33:42+09:00", "last_error_summary": "[h264 @ 0x55fe722619c0] error while decoding MB 23 16, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 37, "h264_decode_error_count": 77, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:33:42+09:00", "last_error_summary": "[h264 @ 0x55fe722619c0] error while decoding MB 23 16, bytestream -7", "last_exit_at": "2026-07-02T16:36:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 137.058, "last_started_at": "2026-07-02T16:33:44+09:00", "pid": 12671, "process": {"exists": false, "pid": 12671}, "restart_count": 49, "restart_count_per_hour": 5, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 50}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 209.746, "sample_count": 1900, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 224792, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:36:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T16:36:01+09:00", "last_frame_read_age_sec": 0.179, "last_frame_read_at": "2026-07-02T16:36:01+09:00", "mjpeg": {"age_sec": 0.175, "exists": true, "mtime": "2026-07-02T16:36:01.704031+09:00", "mtime_epoch": 1782977761.7040308, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188237}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:36:01+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1390.734, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 143, "pushed_frames": 143, "read_frames": 142, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.767, "pushed": 4.767, "read": 4.734}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.591, "count": 142, "max_ms": 16.003, "p95_ms": 13.247}, "gstreamer_push": {"avg_ms": 0.902, "count": 143, "max_ms": 1.118, "p95_ms": 1.045}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.823, "count": 142, "max_ms": 81.201, "p95_ms": 69.16}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.16, "count": 142, "max_ms": 106.97, "p95_ms": 91.413}, "yolo11_640_fire_11class": {"avg_ms": 52.567, "count": 142, "max_ms": 80.277, "p95_ms": 70.83}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.017, "count": 142, "max_ms": 98.627, "p95_ms": 87.639}}, "truncated_count": 0}, "processing_total": {"avg_ms": 200.924, "count": 143, "max_ms": 903.939, "p95_ms": 246.659}, "read_interval": {"avg_ms": 211.414, "count": 142, "max_ms": 914.175, "p95_ms": 256.612}, "triton_wait": {"avg_ms": 67.425, "count": 142, "max_ms": 106.981, "p95_ms": 93.874}, "visualize": {"avg_ms": 10.728, "count": 142, "max_ms": 16.963, "p95_ms": 16.253}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1873}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5353.633, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 37, "h264_decode_error_count": 77, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:33:42+09:00", "last_error_summary": "[h264 @ 0x55fe722619c0] error while decoding MB 23 16, bytestream -7", "last_exit_at": "2026-07-02T16:36:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 137.062, "last_started_at": "2026-07-02T16:33:44+09:00", "pid": 12671, "process": {"exists": false, "pid": 12671}, "restart_count": 49, "restart_count_per_hour": 5, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 50}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 209.746, "sample_count": 1900, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 224792, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:36:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T16:36:01+09:00", "last_frame_read_age_sec": 0.183, "last_frame_read_at": "2026-07-02T16:36:01+09:00", "mjpeg": {"age_sec": 0.178, "exists": true, "mtime": "2026-07-02T16:36:01.704031+09:00", "mtime_epoch": 1782977761.7040308, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188237}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:36:01+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1390.734, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 142, "pushed_frames": 143, "read_frames": 142, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.733, "pushed": 4.767, "read": 4.733}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.591, "count": 142, "max_ms": 16.003, "p95_ms": 13.247}, "gstreamer_push": {"avg_ms": 0.902, "count": 143, "max_ms": 1.118, "p95_ms": 1.045}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.823, "count": 142, "max_ms": 81.201, "p95_ms": 69.16}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.16, "count": 142, "max_ms": 106.97, "p95_ms": 91.413}, "yolo11_640_fire_11class": {"avg_ms": 52.567, "count": 142, "max_ms": 80.277, "p95_ms": 70.83}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.017, "count": 142, "max_ms": 98.627, "p95_ms": 87.639}}, "truncated_count": 0}, "processing_total": {"avg_ms": 200.909, "count": 142, "max_ms": 903.939, "p95_ms": 246.659}, "read_interval": {"avg_ms": 211.414, "count": 142, "max_ms": 914.175, "p95_ms": 256.612}, "triton_wait": {"avg_ms": 67.425, "count": 142, "max_ms": 106.981, "p95_ms": 93.874}, "visualize": {"avg_ms": 10.728, "count": 142, "max_ms": 16.963, "p95_ms": 16.253}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1873}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5461.664, "limit_mb": 8796093022207.996}, "detail": {"pid": 12838, "restart_count": 50, "start_count": 51}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 37, "h264_decode_error_count": 77, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:33:42+09:00", "last_error_summary": "[h264 @ 0x55fe722619c0] error while decoding MB 23 16, bytestream -7", "last_exit_at": "2026-07-02T16:36:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T16:36:04+09:00", "pid": 12838, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.25, "name": "ffmpeg", "pid": 12838, "rss_mb": 139.25, "state": "S (sleeping)", "threads": 106}, "restart_count": 50, "restart_count_per_hour": 6, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 51}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 210.439, "sample_count": 1869, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 224792, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:36:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.141, "last_frame_pushed_at": "2026-07-02T16:36:01+09:00", "last_frame_read_age_sec": 4.316, "last_frame_read_at": "2026-07-02T16:36:01+09:00", "mjpeg": {"age_sec": 4.128, "exists": true, "mtime": "2026-07-02T16:36:01.888035+09:00", "mtime_epoch": 1782977761.8880346, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 188341}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:36:06+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1390.734, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 127, "pushed_frames": 127, "read_frames": 126, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.25, "pushed": 4.25, "read": 4.216}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 10.501, "count": 126, "max_ms": 16.003, "p95_ms": 13.247}, "gstreamer_push": {"avg_ms": 0.908, "count": 127, "max_ms": 1.118, "p95_ms": 1.057}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.942, "count": 126, "max_ms": 81.201, "p95_ms": 69.458}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.033, "count": 126, "max_ms": 106.97, "p95_ms": 86.425}, "yolo11_640_fire_11class": {"avg_ms": 52.497, "count": 126, "max_ms": 80.277, "p95_ms": 69.242}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.886, "count": 126, "max_ms": 98.627, "p95_ms": 87.639}}, "truncated_count": 0}, "processing_total": {"avg_ms": 194.243, "count": 127, "max_ms": 357.201, "p95_ms": 233.181}, "read_interval": {"avg_ms": 204.757, "count": 126, "max_ms": 367.405, "p95_ms": 245.672}, "triton_wait": {"avg_ms": 67.349, "count": 126, "max_ms": 106.981, "p95_ms": 93.874}, "visualize": {"avg_ms": 10.812, "count": 126, "max_ms": 16.963, "p95_ms": 16.266}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.884, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1879}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5357.598, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55c82aad20c0] error while decoding MB 79 31, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 38, "h264_decode_error_count": 78, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:38:16+09:00", "last_error_summary": "[h264 @ 0x55c82aad20c0] error while decoding MB 79 31, bytestream -7", "last_exit_at": "2026-07-02T16:36:04+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.085, "last_started_at": "2026-07-02T16:36:04+09:00", "pid": 12838, "process": {"exists": false, "pid": 12838}, "restart_count": 50, "restart_count_per_hour": 6, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 51}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 216.095, "sample_count": 1673, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 225549, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:38:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-02T16:38:16+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-02T16:38:16+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-02T16:38:16.054769+09:00", "mtime_epoch": 1782977896.0547686, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200130}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:38:16+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1390.992, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 156, "pushed_frames": 156, "read_frames": 156, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.216, "pushed": 5.216, "read": 5.216}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.387, "count": 156, "max_ms": 14.688, "p95_ms": 10.788}, "gstreamer_push": {"avg_ms": 0.896, "count": 156, "max_ms": 1.396, "p95_ms": 1.039}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.954, "count": 156, "max_ms": 82.479, "p95_ms": 63.81}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.224, "count": 156, "max_ms": 110.986, "p95_ms": 89.635}, "yolo11_640_fire_11class": {"avg_ms": 51.18, "count": 156, "max_ms": 85.525, "p95_ms": 68.931}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.85, "count": 156, "max_ms": 94.413, "p95_ms": 83.922}}, "truncated_count": 0}, "processing_total": {"avg_ms": 183.027, "count": 156, "max_ms": 261.489, "p95_ms": 213.569}, "read_interval": {"avg_ms": 192.547, "count": 156, "max_ms": 272.618, "p95_ms": 223.505}, "triton_wait": {"avg_ms": 66.219, "count": 156, "max_ms": 110.998, "p95_ms": 89.647}, "visualize": {"avg_ms": 8.265, "count": 156, "max_ms": 12.814, "p95_ms": 9.868}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.906, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1879}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5360.41, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T16:38:16+09:00", "last_error_summary": "[h264 @ 0x55c82aad20c0] error while decoding MB 79 31, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 38, "h264_decode_error_count": 78, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:38:16+09:00", "last_error_summary": "[h264 @ 0x55c82aad20c0] error while decoding MB 79 31, bytestream -7", "last_exit_at": "2026-07-02T16:38:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.205, "last_started_at": "2026-07-02T16:36:04+09:00", "pid": 12838, "process": {"exists": false, "pid": 12838}, "restart_count": 50, "restart_count_per_hour": 6, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 51}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 216.095, "sample_count": 1674, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 225550, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:38:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T16:38:16+09:00", "last_frame_read_age_sec": 0.174, "last_frame_read_at": "2026-07-02T16:38:16+09:00", "mjpeg": {"age_sec": 0.165, "exists": true, "mtime": "2026-07-02T16:38:16.054769+09:00", "mtime_epoch": 1782977896.0547686, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200130}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:38:16+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1390.992, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 156, "pushed_frames": 156, "read_frames": 155, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.228, "pushed": 5.228, "read": 5.195}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.394, "count": 156, "max_ms": 14.688, "p95_ms": 10.788}, "gstreamer_push": {"avg_ms": 0.896, "count": 156, "max_ms": 1.396, "p95_ms": 1.039}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.967, "count": 156, "max_ms": 82.479, "p95_ms": 63.81}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.071, "count": 156, "max_ms": 110.986, "p95_ms": 89.635}, "yolo11_640_fire_11class": {"avg_ms": 51.1, "count": 156, "max_ms": 85.525, "p95_ms": 68.931}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.698, "count": 156, "max_ms": 94.413, "p95_ms": 83.922}}, "truncated_count": 0}, "processing_total": {"avg_ms": 182.924, "count": 156, "max_ms": 261.489, "p95_ms": 213.569}, "read_interval": {"avg_ms": 192.541, "count": 155, "max_ms": 272.618, "p95_ms": 223.505}, "triton_wait": {"avg_ms": 66.066, "count": 156, "max_ms": 110.998, "p95_ms": 89.647}, "visualize": {"avg_ms": 8.281, "count": 156, "max_ms": 12.814, "p95_ms": 9.978}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.838, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1879}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5360.402, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 38, "h264_decode_error_count": 78, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:38:16+09:00", "last_error_summary": "[h264 @ 0x55c82aad20c0] error while decoding MB 79 31, bytestream -7", "last_exit_at": "2026-07-02T16:38:16+09:00", "last_returncode": 0, "last_stable_duration_sec": 132.209, "last_started_at": "2026-07-02T16:36:04+09:00", "pid": 12838, "process": {"exists": false, "pid": 12838}, "restart_count": 50, "restart_count_per_hour": 6, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 51}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 216.095, "sample_count": 1674, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 225550, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:38:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T16:38:16+09:00", "last_frame_read_age_sec": 0.178, "last_frame_read_at": "2026-07-02T16:38:16+09:00", "mjpeg": {"age_sec": 0.169, "exists": true, "mtime": "2026-07-02T16:38:16.054769+09:00", "mtime_epoch": 1782977896.0547686, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200130}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:38:16+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1390.992, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 156, "pushed_frames": 156, "read_frames": 155, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.228, "pushed": 5.228, "read": 5.194}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.394, "count": 156, "max_ms": 14.688, "p95_ms": 10.788}, "gstreamer_push": {"avg_ms": 0.896, "count": 156, "max_ms": 1.396, "p95_ms": 1.039}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.967, "count": 156, "max_ms": 82.479, "p95_ms": 63.81}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.071, "count": 156, "max_ms": 110.986, "p95_ms": 89.635}, "yolo11_640_fire_11class": {"avg_ms": 51.1, "count": 156, "max_ms": 85.525, "p95_ms": 68.931}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.698, "count": 156, "max_ms": 94.413, "p95_ms": 83.922}}, "truncated_count": 0}, "processing_total": {"avg_ms": 182.924, "count": 156, "max_ms": 261.489, "p95_ms": 213.569}, "read_interval": {"avg_ms": 192.541, "count": 155, "max_ms": 272.618, "p95_ms": 223.505}, "triton_wait": {"avg_ms": 66.066, "count": 156, "max_ms": 110.998, "p95_ms": 89.647}, "visualize": {"avg_ms": 8.281, "count": 156, "max_ms": 12.814, "p95_ms": 9.978}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.842, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1879}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5473.52, "limit_mb": 8796093022207.996}, "detail": {"pid": 12945, "restart_count": 51, "start_count": 52}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 38, "h264_decode_error_count": 78, "h264_decode_error_count_per_hour": 10, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:38:16+09:00", "last_error_summary": "[h264 @ 0x55c82aad20c0] error while decoding MB 79 31, bytestream -7", "last_exit_at": "2026-07-02T16:38:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T16:38:18+09:00", "pid": 12945, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.293, "name": "ffmpeg", "pid": 12945, "rss_mb": 139.293, "state": "S (sleeping)", "threads": 106}, "restart_count": 51, "restart_count_per_hour": 7, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 52}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 216.286, "sample_count": 1643, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 225550, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:38:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.711, "last_frame_pushed_at": "2026-07-02T16:38:16+09:00", "last_frame_read_age_sec": 4.882, "last_frame_read_at": "2026-07-02T16:38:16+09:00", "mjpeg": {"age_sec": 4.694, "exists": true, "mtime": "2026-07-02T16:38:16.234772+09:00", "mtime_epoch": 1782977896.234772, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200385}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:38:20+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1390.992, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 133, "pushed_frames": 133, "read_frames": 132, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.441, "pushed": 4.441, "read": 4.408}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.434, "count": 132, "max_ms": 14.688, "p95_ms": 11.044}, "gstreamer_push": {"avg_ms": 0.897, "count": 133, "max_ms": 1.39, "p95_ms": 1.039}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.454, "count": 132, "max_ms": 71.199, "p95_ms": 61.85}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.713, "count": 132, "max_ms": 103.956, "p95_ms": 85.777}, "yolo11_640_fire_11class": {"avg_ms": 50.477, "count": 132, "max_ms": 79.883, "p95_ms": 67.608}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.779, "count": 132, "max_ms": 92.113, "p95_ms": 82.583}}, "truncated_count": 0}, "processing_total": {"avg_ms": 181.718, "count": 133, "max_ms": 261.489, "p95_ms": 209.793}, "read_interval": {"avg_ms": 191.261, "count": 132, "max_ms": 272.618, "p95_ms": 217.973}, "triton_wait": {"avg_ms": 64.95, "count": 132, "max_ms": 103.968, "p95_ms": 86.599}, "visualize": {"avg_ms": 8.246, "count": 132, "max_ms": 12.814, "p95_ms": 9.76}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.948, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1885}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5365.859, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55bf758acec0] error while decoding MB 8 39, bytestream -14", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 39, "h264_decode_error_count": 79, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:40:32+09:00", "last_error_summary": "[h264 @ 0x55bf758acec0] error while decoding MB 8 39, bytestream -14", "last_exit_at": "2026-07-02T16:38:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 133.721, "last_started_at": "2026-07-02T16:38:18+09:00", "pid": 12945, "process": {"exists": false, "pid": 12945}, "restart_count": 51, "restart_count_per_hour": 7, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 52}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 218.548, "sample_count": 1586, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 226239, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:40:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.071, "last_frame_pushed_at": "2026-07-02T16:40:32+09:00", "last_frame_read_age_sec": 0.063, "last_frame_read_at": "2026-07-02T16:40:32+09:00", "mjpeg": {"age_sec": 0.058, "exists": true, "mtime": "2026-07-02T16:40:32.593431+09:00", "mtime_epoch": 1782978032.5934315, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203079}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:40:32+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1391.508, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 150, "pushed_frames": 150, "read_frames": 150, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.012, "pushed": 5.012, "read": 5.012}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.629, "count": 149, "max_ms": 12.823, "p95_ms": 11.036}, "gstreamer_push": {"avg_ms": 0.875, "count": 150, "max_ms": 1.319, "p95_ms": 0.954}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.519, "count": 149, "max_ms": 79.126, "p95_ms": 71.566}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.399, "count": 149, "max_ms": 102.928, "p95_ms": 88.886}, "yolo11_640_fire_11class": {"avg_ms": 53.078, "count": 149, "max_ms": 81.695, "p95_ms": 71.806}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.186, "count": 149, "max_ms": 100.429, "p95_ms": 89.549}}, "truncated_count": 0}, "processing_total": {"avg_ms": 190.881, "count": 150, "max_ms": 228.401, "p95_ms": 218.035}, "read_interval": {"avg_ms": 200.535, "count": 150, "max_ms": 238.924, "p95_ms": 228.836}, "triton_wait": {"avg_ms": 67.897, "count": 149, "max_ms": 102.943, "p95_ms": 93.777}, "visualize": {"avg_ms": 8.571, "count": 149, "max_ms": 11.278, "p95_ms": 9.72}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.93, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1885}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5356.785, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T16:40:32+09:00", "last_error_summary": "[h264 @ 0x55bf758acec0] error while decoding MB 8 39, bytestream -14", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 39, "h264_decode_error_count": 79, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:40:32+09:00", "last_error_summary": "[h264 @ 0x55bf758acec0] error while decoding MB 8 39, bytestream -14", "last_exit_at": "2026-07-02T16:40:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 133.853, "last_started_at": "2026-07-02T16:38:18+09:00", "pid": 12945, "process": {"exists": false, "pid": 12945}, "restart_count": 51, "restart_count_per_hour": 7, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 52}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 218.548, "sample_count": 1586, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 226240, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:40:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T16:40:32+09:00", "last_frame_read_age_sec": 0.195, "last_frame_read_at": "2026-07-02T16:40:32+09:00", "mjpeg": {"age_sec": 0.187, "exists": true, "mtime": "2026-07-02T16:40:32.593431+09:00", "mtime_epoch": 1782978032.5934315, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203079}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:40:32+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1391.508, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 150, "pushed_frames": 150, "read_frames": 149, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.026, "pushed": 5.026, "read": 4.992}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.658, "count": 150, "max_ms": 13.962, "p95_ms": 11.227}, "gstreamer_push": {"avg_ms": 0.872, "count": 150, "max_ms": 1.315, "p95_ms": 0.948}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.485, "count": 150, "max_ms": 79.126, "p95_ms": 71.566}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.353, "count": 150, "max_ms": 102.928, "p95_ms": 88.886}, "yolo11_640_fire_11class": {"avg_ms": 53.035, "count": 150, "max_ms": 81.695, "p95_ms": 71.806}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.124, "count": 150, "max_ms": 100.429, "p95_ms": 89.549}}, "truncated_count": 0}, "processing_total": {"avg_ms": 190.705, "count": 150, "max_ms": 228.401, "p95_ms": 218.035}, "read_interval": {"avg_ms": 200.356, "count": 149, "max_ms": 238.924, "p95_ms": 228.836}, "triton_wait": {"avg_ms": 67.822, "count": 150, "max_ms": 102.943, "p95_ms": 93.777}, "visualize": {"avg_ms": 8.588, "count": 150, "max_ms": 11.278, "p95_ms": 9.758}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.845, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1885}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5356.777, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 39, "h264_decode_error_count": 79, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:40:32+09:00", "last_error_summary": "[h264 @ 0x55bf758acec0] error while decoding MB 8 39, bytestream -14", "last_exit_at": "2026-07-02T16:40:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 133.857, "last_started_at": "2026-07-02T16:38:18+09:00", "pid": 12945, "process": {"exists": false, "pid": 12945}, "restart_count": 51, "restart_count_per_hour": 7, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 52}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 218.548, "sample_count": 1586, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 226240, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:40:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T16:40:32+09:00", "last_frame_read_age_sec": 0.199, "last_frame_read_at": "2026-07-02T16:40:32+09:00", "mjpeg": {"age_sec": 0.191, "exists": true, "mtime": "2026-07-02T16:40:32.593431+09:00", "mtime_epoch": 1782978032.5934315, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203079}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:40:32+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1391.508, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 150, "pushed_frames": 150, "read_frames": 149, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.025, "pushed": 5.025, "read": 4.992}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.658, "count": 150, "max_ms": 13.962, "p95_ms": 11.227}, "gstreamer_push": {"avg_ms": 0.872, "count": 150, "max_ms": 1.315, "p95_ms": 0.948}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.485, "count": 150, "max_ms": 79.126, "p95_ms": 71.566}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.353, "count": 150, "max_ms": 102.928, "p95_ms": 88.886}, "yolo11_640_fire_11class": {"avg_ms": 53.035, "count": 150, "max_ms": 81.695, "p95_ms": 71.806}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.124, "count": 150, "max_ms": 100.429, "p95_ms": 89.549}}, "truncated_count": 0}, "processing_total": {"avg_ms": 190.705, "count": 150, "max_ms": 228.401, "p95_ms": 218.035}, "read_interval": {"avg_ms": 200.356, "count": 149, "max_ms": 238.924, "p95_ms": 228.836}, "triton_wait": {"avg_ms": 67.822, "count": 150, "max_ms": 102.943, "p95_ms": 93.777}, "visualize": {"avg_ms": 8.588, "count": 150, "max_ms": 11.278, "p95_ms": 9.758}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.849, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1885}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5436.594, "limit_mb": 8796093022207.996}, "detail": {"pid": 13052, "restart_count": 52, "start_count": 53}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 39, "h264_decode_error_count": 79, "h264_decode_error_count_per_hour": 11, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:40:32+09:00", "last_error_summary": "[h264 @ 0x55bf758acec0] error while decoding MB 8 39, bytestream -14", "last_exit_at": "2026-07-02T16:40:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T16:40:35+09:00", "pid": 13052, "process": {"exists": true, "fd_count": 5, "hwm_mb": 104.273, "name": "ffmpeg", "pid": 13052, "rss_mb": 104.273, "state": "S (sleeping)", "threads": 49}, "restart_count": 52, "restart_count_per_hour": 8, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 53}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 216.561, "sample_count": 1569, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 226240, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:40:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.559, "last_frame_pushed_at": "2026-07-02T16:40:32+09:00", "last_frame_read_age_sec": 4.751, "last_frame_read_at": "2026-07-02T16:40:32+09:00", "mjpeg": {"age_sec": 4.543, "exists": true, "mtime": "2026-07-02T16:40:32.793435+09:00", "mtime_epoch": 1782978032.7934353, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 203053}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:40:37+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1391.508, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 128, "pushed_frames": 128, "read_frames": 127, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.272, "pushed": 4.272, "read": 4.239}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.64, "count": 127, "max_ms": 13.962, "p95_ms": 11.227}, "gstreamer_push": {"avg_ms": 0.872, "count": 128, "max_ms": 1.315, "p95_ms": 0.954}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.73, "count": 127, "max_ms": 79.126, "p95_ms": 72.373}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.624, "count": 127, "max_ms": 102.928, "p95_ms": 91.488}, "yolo11_640_fire_11class": {"avg_ms": 53.429, "count": 127, "max_ms": 81.695, "p95_ms": 76.296}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.532, "count": 127, "max_ms": 100.429, "p95_ms": 89.873}}, "truncated_count": 0}, "processing_total": {"avg_ms": 190.458, "count": 128, "max_ms": 228.401, "p95_ms": 219.638}, "read_interval": {"avg_ms": 200.074, "count": 127, "max_ms": 238.924, "p95_ms": 229.528}, "triton_wait": {"avg_ms": 68.261, "count": 127, "max_ms": 102.943, "p95_ms": 94.043}, "visualize": {"avg_ms": 8.541, "count": 127, "max_ms": 11.278, "p95_ms": 9.653}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.959, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1890}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5363.664, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x556a970ea840] error while decoding MB 16 31, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 40, "h264_decode_error_count": 80, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:42:43+09:00", "last_error_summary": "[h264 @ 0x556a970ea840] error while decoding MB 16 31, bytestream -5", "last_exit_at": "2026-07-02T16:40:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.031, "last_started_at": "2026-07-02T16:40:35+09:00", "pid": 13052, "process": {"exists": false, "pid": 13052}, "restart_count": 52, "restart_count_per_hour": 8, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 53}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 228.401, "sample_count": 1489, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 226868, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:42:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.067, "last_frame_pushed_at": "2026-07-02T16:42:43+09:00", "last_frame_read_age_sec": 0.055, "last_frame_read_at": "2026-07-02T16:42:43+09:00", "mjpeg": {"age_sec": 0.055, "exists": true, "mtime": "2026-07-02T16:42:43.311894+09:00", "mtime_epoch": 1782978163.3118942, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197390}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:42:43+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1391.766, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 159, "pushed_frames": 159, "read_frames": 159, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.325, "pushed": 5.325, "read": 5.325}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.599, "count": 158, "max_ms": 13.743, "p95_ms": 12.455}, "gstreamer_push": {"avg_ms": 0.915, "count": 159, "max_ms": 1.553, "p95_ms": 1.079}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.042, "count": 158, "max_ms": 80.584, "p95_ms": 72.231}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.835, "count": 158, "max_ms": 106.147, "p95_ms": 91.883}, "yolo11_640_fire_11class": {"avg_ms": 51.194, "count": 158, "max_ms": 79.086, "p95_ms": 71.487}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.966, "count": 158, "max_ms": 107.913, "p95_ms": 97.06}}, "truncated_count": 0}, "processing_total": {"avg_ms": 179.144, "count": 159, "max_ms": 272.814, "p95_ms": 240.942}, "read_interval": {"avg_ms": 189.0, "count": 159, "max_ms": 282.495, "p95_ms": 249.9}, "triton_wait": {"avg_ms": 66.601, "count": 158, "max_ms": 107.926, "p95_ms": 98.007}, "visualize": {"avg_ms": 7.956, "count": 158, "max_ms": 11.944, "p95_ms": 11.076}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.859, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1890}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5365.969, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T16:42:43+09:00", "last_error_summary": "[h264 @ 0x556a970ea840] error while decoding MB 16 31, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 40, "h264_decode_error_count": 80, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:42:43+09:00", "last_error_summary": "[h264 @ 0x556a970ea840] error while decoding MB 16 31, bytestream -5", "last_exit_at": "2026-07-02T16:42:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.105, "last_started_at": "2026-07-02T16:40:35+09:00", "pid": 13052, "process": {"exists": false, "pid": 13052}, "restart_count": 52, "restart_count_per_hour": 8, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 53}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 228.401, "sample_count": 1490, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 226869, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:42:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T16:42:43+09:00", "last_frame_read_age_sec": 0.13, "last_frame_read_at": "2026-07-02T16:42:43+09:00", "mjpeg": {"age_sec": 0.129, "exists": true, "mtime": "2026-07-02T16:42:43.311894+09:00", "mtime_epoch": 1782978163.3118942, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197390}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:42:43+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1391.766, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 160, "pushed_frames": 160, "read_frames": 159, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.345, "pushed": 5.345, "read": 5.312}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.607, "count": 159, "max_ms": 13.743, "p95_ms": 12.455}, "gstreamer_push": {"avg_ms": 0.916, "count": 160, "max_ms": 1.553, "p95_ms": 1.067}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.025, "count": 159, "max_ms": 80.584, "p95_ms": 72.231}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.825, "count": 159, "max_ms": 106.147, "p95_ms": 91.883}, "yolo11_640_fire_11class": {"avg_ms": 51.196, "count": 159, "max_ms": 79.086, "p95_ms": 71.487}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.931, "count": 159, "max_ms": 107.913, "p95_ms": 97.06}}, "truncated_count": 0}, "processing_total": {"avg_ms": 178.789, "count": 160, "max_ms": 272.814, "p95_ms": 237.344}, "read_interval": {"avg_ms": 189.0, "count": 159, "max_ms": 282.495, "p95_ms": 249.9}, "triton_wait": {"avg_ms": 66.561, "count": 159, "max_ms": 107.926, "p95_ms": 98.007}, "visualize": {"avg_ms": 7.938, "count": 159, "max_ms": 11.944, "p95_ms": 11.076}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.933, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1890}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5365.777, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 40, "h264_decode_error_count": 80, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:42:43+09:00", "last_error_summary": "[h264 @ 0x556a970ea840] error while decoding MB 16 31, bytestream -5", "last_exit_at": "2026-07-02T16:42:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 128.112, "last_started_at": "2026-07-02T16:40:35+09:00", "pid": 13052, "process": {"exists": false, "pid": 13052}, "restart_count": 52, "restart_count_per_hour": 8, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 53}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 228.401, "sample_count": 1490, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 226869, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:42:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-02T16:42:43+09:00", "last_frame_read_age_sec": 0.137, "last_frame_read_at": "2026-07-02T16:42:43+09:00", "mjpeg": {"age_sec": 0.137, "exists": true, "mtime": "2026-07-02T16:42:43.311894+09:00", "mtime_epoch": 1782978163.3118942, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 197390}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:42:43+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1391.766, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 160, "pushed_frames": 160, "read_frames": 159, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.344, "pushed": 5.344, "read": 5.311}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.607, "count": 159, "max_ms": 13.743, "p95_ms": 12.455}, "gstreamer_push": {"avg_ms": 0.916, "count": 160, "max_ms": 1.553, "p95_ms": 1.067}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.025, "count": 159, "max_ms": 80.584, "p95_ms": 72.231}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.825, "count": 159, "max_ms": 106.147, "p95_ms": 91.883}, "yolo11_640_fire_11class": {"avg_ms": 51.196, "count": 159, "max_ms": 79.086, "p95_ms": 71.487}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 64.931, "count": 159, "max_ms": 107.913, "p95_ms": 97.06}}, "truncated_count": 0}, "processing_total": {"avg_ms": 178.789, "count": 160, "max_ms": 272.814, "p95_ms": 237.344}, "read_interval": {"avg_ms": 189.0, "count": 159, "max_ms": 282.495, "p95_ms": 249.9}, "triton_wait": {"avg_ms": 66.561, "count": 159, "max_ms": 107.926, "p95_ms": 98.007}, "visualize": {"avg_ms": 7.938, "count": 159, "max_ms": 11.944, "p95_ms": 11.076}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.94, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1890}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5474.805, "limit_mb": 8796093022207.996}, "detail": {"pid": 13159, "restart_count": 53, "start_count": 54}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 40, "h264_decode_error_count": 80, "h264_decode_error_count_per_hour": 12, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:42:43+09:00", "last_error_summary": "[h264 @ 0x556a970ea840] error while decoding MB 16 31, bytestream -5", "last_exit_at": "2026-07-02T16:42:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T16:42:46+09:00", "pid": 13159, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.484, "name": "ffmpeg", "pid": 13159, "rss_mb": 139.484, "state": "S (sleeping)", "threads": 106}, "restart_count": 53, "restart_count_per_hour": 9, "rtsp_timeout_count": 22, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 54}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 228.601, "sample_count": 1466, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 226869, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:42:43+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.69, "last_frame_pushed_at": "2026-07-02T16:42:43+09:00", "last_frame_read_age_sec": 4.814, "last_frame_read_at": "2026-07-02T16:42:43+09:00", "mjpeg": {"age_sec": 4.673, "exists": true, "mtime": "2026-07-02T16:42:43.451897+09:00", "mtime_epoch": 1782978163.451897, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196707}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:42:48+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1391.766, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 137, "pushed_frames": 137, "read_frames": 136, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.589, "pushed": 4.589, "read": 4.556}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 9.429, "count": 137, "max_ms": 13.743, "p95_ms": 11.955}, "gstreamer_push": {"avg_ms": 0.923, "count": 137, "max_ms": 1.553, "p95_ms": 1.081}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.32, "count": 137, "max_ms": 80.584, "p95_ms": 71.741}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.107, "count": 137, "max_ms": 106.147, "p95_ms": 92.206}, "yolo11_640_fire_11class": {"avg_ms": 51.893, "count": 137, "max_ms": 79.086, "p95_ms": 72.166}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 65.15, "count": 137, "max_ms": 107.913, "p95_ms": 97.06}}, "truncated_count": 0}, "processing_total": {"avg_ms": 174.896, "count": 137, "max_ms": 272.814, "p95_ms": 223.599}, "read_interval": {"avg_ms": 185.335, "count": 136, "max_ms": 282.495, "p95_ms": 233.509}, "triton_wait": {"avg_ms": 66.826, "count": 137, "max_ms": 107.926, "p95_ms": 98.205}, "visualize": {"avg_ms": 7.745, "count": 137, "max_ms": 11.291, "p95_ms": 10.805}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.852, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 38, "tracking_summary": 1909}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5441.887, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55f483e2b600] error while decoding MB 88 48, bytestream -18 [in#0/rtsp @ 0x55f483cbbe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f483cbbe00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 32, "h264_decode_error_count": 33, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:45:09+09:00", "last_error_summary": "[h264 @ 0x55f483e2b600] error while decoding MB 88 48, bytestream -18 [in#0/rtsp @ 0x55f483cbbe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f483cbbe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T16:27:22+09:00", "last_returncode": 0, "last_stable_duration_sec": 1066.783, "last_started_at": "2026-07-02T16:27:22+09:00", "pid": 12459, "process": {"exists": false, "pid": 12459}, "restart_count": 31, "restart_count_per_hour": 2, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 32}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 252.022, "sample_count": 2085, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 229189, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:45:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.073, "last_frame_pushed_at": "2026-07-02T16:45:09+09:00", "last_frame_read_age_sec": 0.062, "last_frame_read_at": "2026-07-02T16:45:09+09:00", "mjpeg": {"age_sec": 0.054, "exists": true, "mtime": "2026-07-02T16:45:09.578573+09:00", "mtime_epoch": 1782978309.5785732, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 250360}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T16:45:09+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1431.711, "name": "celery", "pid": 267, "rss_mb": 1268.629, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 12, "pushed_frames": 12, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.083, "pushed": 1.083, "read": 1.173}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.84, "count": 12, "max_ms": 9.194, "p95_ms": 9.194}, "gstreamer_push": {"avg_ms": 1.263, "count": 12, "max_ms": 1.57, "p95_ms": 1.57}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.727, "count": 12, "max_ms": 53.799, "p95_ms": 53.799}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 47.37, "count": 12, "max_ms": 53.113, "p95_ms": 53.113}, "yolo11_640_fire_11class": {"avg_ms": 49.771, "count": 12, "max_ms": 56.015, "p95_ms": 56.015}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.225, "count": 12, "max_ms": 54.371, "p95_ms": 54.371}}, "truncated_count": 0}, "processing_total": {"avg_ms": 65.271, "count": 12, "max_ms": 72.601, "p95_ms": 72.601}, "read_interval": {"avg_ms": 2321.368, "count": 13, "max_ms": 19159.791, "p95_ms": 19159.791}, "triton_wait": {"avg_ms": 49.917, "count": 12, "max_ms": 56.028, "p95_ms": 56.028}, "visualize": {"avg_ms": 3.67, "count": 12, "max_ms": 4.572, "p95_ms": 4.572}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 11.08, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 38, "tracking_summary": 1909}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5441.652, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T16:45:09+09:00", "last_error_summary": "[h264 @ 0x55f483e2b600] error while decoding MB 88 48, bytestream -18 [in#0/rtsp @ 0x55f483cbbe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f483cbbe00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 32, "h264_decode_error_count": 33, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:45:09+09:00", "last_error_summary": "[h264 @ 0x55f483e2b600] error while decoding MB 88 48, bytestream -18 [in#0/rtsp @ 0x55f483cbbe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f483cbbe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T16:45:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 1066.812, "last_started_at": "2026-07-02T16:27:22+09:00", "pid": 12459, "process": {"exists": false, "pid": 12459}, "restart_count": 31, "restart_count_per_hour": 2, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 32}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 252.022, "sample_count": 2086, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 229190, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:45:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T16:45:09+09:00", "last_frame_read_age_sec": 0.091, "last_frame_read_at": "2026-07-02T16:45:09+09:00", "mjpeg": {"age_sec": 0.083, "exists": true, "mtime": "2026-07-02T16:45:09.578573+09:00", "mtime_epoch": 1782978309.5785732, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 250360}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T16:45:09+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1431.711, "name": "celery", "pid": 267, "rss_mb": 1268.629, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.17, "pushed": 1.17, "read": 1.17}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.826, "count": 13, "max_ms": 9.194, "p95_ms": 9.194}, "gstreamer_push": {"avg_ms": 1.265, "count": 13, "max_ms": 1.57, "p95_ms": 1.57}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.485, "count": 13, "max_ms": 53.799, "p95_ms": 53.799}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 48.642, "count": 13, "max_ms": 63.9, "p95_ms": 63.9}, "yolo11_640_fire_11class": {"avg_ms": 49.822, "count": 13, "max_ms": 56.015, "p95_ms": 56.015}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.52, "count": 13, "max_ms": 54.371, "p95_ms": 54.371}}, "truncated_count": 0}, "processing_total": {"avg_ms": 66.659, "count": 13, "max_ms": 83.325, "p95_ms": 83.325}, "read_interval": {"avg_ms": 2321.368, "count": 13, "max_ms": 19159.791, "p95_ms": 19159.791}, "triton_wait": {"avg_ms": 50.994, "count": 13, "max_ms": 63.911, "p95_ms": 63.911}, "visualize": {"avg_ms": 3.987, "count": 13, "max_ms": 7.801, "p95_ms": 7.801}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 11.109, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 38, "tracking_summary": 1909}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5441.645, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 32, "h264_decode_error_count": 33, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:45:09+09:00", "last_error_summary": "[h264 @ 0x55f483e2b600] error while decoding MB 88 48, bytestream -18 [in#0/rtsp @ 0x55f483cbbe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f483cbbe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T16:45:09+09:00", "last_returncode": 0, "last_stable_duration_sec": 1066.819, "last_started_at": "2026-07-02T16:27:22+09:00", "pid": 12459, "process": {"exists": false, "pid": 12459}, "restart_count": 31, "restart_count_per_hour": 2, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 32}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 252.022, "sample_count": 2085, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 229190, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:45:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-02T16:45:09+09:00", "last_frame_read_age_sec": 0.098, "last_frame_read_at": "2026-07-02T16:45:09+09:00", "mjpeg": {"age_sec": 0.09, "exists": true, "mtime": "2026-07-02T16:45:09.578573+09:00", "mtime_epoch": 1782978309.5785732, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 250360}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T16:45:09+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1431.711, "name": "celery", "pid": 267, "rss_mb": 1268.629, "state": "S (sleeping)", "threads": 121}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 29, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.17, "pushed": 1.17, "read": 1.17}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.826, "count": 13, "max_ms": 9.194, "p95_ms": 9.194}, "gstreamer_push": {"avg_ms": 1.265, "count": 13, "max_ms": 1.57, "p95_ms": 1.57}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.485, "count": 13, "max_ms": 53.799, "p95_ms": 53.799}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 48.642, "count": 13, "max_ms": 63.9, "p95_ms": 63.9}, "yolo11_640_fire_11class": {"avg_ms": 49.822, "count": 13, "max_ms": 56.015, "p95_ms": 56.015}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.52, "count": 13, "max_ms": 54.371, "p95_ms": 54.371}}, "truncated_count": 0}, "processing_total": {"avg_ms": 66.659, "count": 13, "max_ms": 83.325, "p95_ms": 83.325}, "read_interval": {"avg_ms": 2321.368, "count": 13, "max_ms": 19159.791, "p95_ms": 19159.791}, "triton_wait": {"avg_ms": 50.994, "count": 13, "max_ms": 63.911, "p95_ms": 63.911}, "visualize": {"avg_ms": 3.987, "count": 13, "max_ms": 7.801, "p95_ms": 7.801}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 11.116, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 38, "tracking_summary": 1909}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5470.586, "limit_mb": 8796093022207.996}, "detail": {"pid": 13326, "restart_count": 32, "start_count": 33}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 32, "h264_decode_error_count": 33, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:45:09+09:00", "last_error_summary": "[h264 @ 0x55f483e2b600] error while decoding MB 88 48, bytestream -18 [in#0/rtsp @ 0x55f483cbbe00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f483cbbe00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T16:45:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T16:45:12+09:00", "pid": 13326, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.094, "name": "ffmpeg", "pid": 13326, "rss_mb": 58.094, "state": "S (sleeping)", "threads": 1}, "restart_count": 32, "restart_count_per_hour": 3, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 33}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 252.204, "sample_count": 2042, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 229190, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:45:09+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 4.436, "last_frame_pushed_at": "2026-07-02T16:45:09+09:00", "last_frame_read_age_sec": 4.521, "last_frame_read_at": "2026-07-02T16:45:09+09:00", "mjpeg": {"age_sec": 4.418, "exists": true, "mtime": "2026-07-02T16:45:09.674575+09:00", "mtime_epoch": 1782978309.6745749, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 250065}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T16:45:14+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1431.711, "name": "celery", "pid": 267, "rss_mb": 1268.629, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 25, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.837, "pushed": 0.837, "read": 0.837}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.826, "count": 13, "max_ms": 9.194, "p95_ms": 9.194}, "gstreamer_push": {"avg_ms": 1.265, "count": 13, "max_ms": 1.57, "p95_ms": 1.57}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 47.485, "count": 13, "max_ms": 53.799, "p95_ms": 53.799}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 48.642, "count": 13, "max_ms": 63.9, "p95_ms": 63.9}, "yolo11_640_fire_11class": {"avg_ms": 49.822, "count": 13, "max_ms": 56.015, "p95_ms": 56.015}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 48.52, "count": 13, "max_ms": 54.371, "p95_ms": 54.371}}, "truncated_count": 0}, "processing_total": {"avg_ms": 66.659, "count": 13, "max_ms": 83.325, "p95_ms": 83.325}, "read_interval": {"avg_ms": 2321.368, "count": 13, "max_ms": 19159.791, "p95_ms": 19159.791}, "triton_wait": {"avg_ms": 50.994, "count": 13, "max_ms": 63.911, "p95_ms": 63.911}, "visualize": {"avg_ms": 3.987, "count": 13, "max_ms": 7.801, "p95_ms": 7.801}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 15.54, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1901}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5505.211, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x555e8b189840] left block unavailable for requested intra mode [h264 @ 0x555e8b189840] error while decoding MB 0 25, bytestream 5005 [h264 @ 0x555e8baea080] left block unavailable for requested intra mode [h264 @ 0x555e8baea080] error while decoding MB 0 39, bytestream 58008 [h264 @ 0x555e8b078000] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555e8b078000] error while decoding MB 0 18, bytestream 4205 [h264 @ 0x555e8b078000] reference picture missing during reorder [h", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 41, "h264_decode_error_count": 84, "h264_decode_error_count_per_hour": 16, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:46:08+09:00", "last_error_summary": "[h264 @ 0x555e8b189840] left block unavailable for requested intra mode [h264 @ 0x555e8b189840] error while decoding MB 0 25, bytestream 5005 [h264 @ 0x555e8baea080] left block unavailable for requested intra mode [h264 @ 0x555e8baea080] error while decoding MB 0 39, bytestream 58008 [h264 @ 0x555e8b078000] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555e8b078000] error while decoding MB 0 18, bytestream 4205 [h264 @ 0x555e8b078000] reference picture missing during reorder [h", "last_exit_at": "2026-07-02T16:42:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 202.407, "last_started_at": "2026-07-02T16:42:46+09:00", "pid": 13159, "process": {"exists": false, "pid": 13159}, "restart_count": 53, "restart_count_per_hour": 9, "rtsp_timeout_count": 24, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 54}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 219.712, "sample_count": 1784, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 228179, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:46:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-02T16:46:08+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-02T16:46:08+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-02T16:46:08.487634+09:00", "mtime_epoch": 1782978368.4876337, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205032}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:46:08+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1392.281, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 78, "pushed_frames": 78, "read_frames": 78, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.61, "pushed": 2.61, "read": 2.61}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.665, "count": 78, "max_ms": 10.119, "p95_ms": 7.693}, "gstreamer_push": {"avg_ms": 1.064, "count": 78, "max_ms": 1.214, "p95_ms": 1.178}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.026, "count": 78, "max_ms": 73.671, "p95_ms": 64.391}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.686, "count": 78, "max_ms": 94.604, "p95_ms": 88.489}, "yolo11_640_fire_11class": {"avg_ms": 51.129, "count": 78, "max_ms": 74.623, "p95_ms": 68.613}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.407, "count": 78, "max_ms": 90.617, "p95_ms": 82.82}}, "truncated_count": 0}, "processing_total": {"avg_ms": 124.554, "count": 78, "max_ms": 172.084, "p95_ms": 156.059}, "read_interval": {"avg_ms": 384.169, "count": 78, "max_ms": 19453.062, "p95_ms": 172.928}, "triton_wait": {"avg_ms": 64.854, "count": 78, "max_ms": 94.616, "p95_ms": 89.227}, "visualize": {"avg_ms": 5.37, "count": 78, "max_ms": 6.767, "p95_ms": 6.473}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.885, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1901}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5504.141, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T16:46:08+09:00", "last_error_summary": "[h264 @ 0x555e8b189840] left block unavailable for requested intra mode [h264 @ 0x555e8b189840] error while decoding MB 0 25, bytestream 5005 [h264 @ 0x555e8baea080] left block unavailable for requested intra mode [h264 @ 0x555e8baea080] error while decoding MB 0 39, bytestream 58008 [h264 @ 0x555e8b078000] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555e8b078000] error while decoding MB 0 18, bytestream 4205 [h264 @ 0x555e8b078000] reference picture missing during reorder [h", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 41, "h264_decode_error_count": 84, "h264_decode_error_count_per_hour": 16, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:46:08+09:00", "last_error_summary": "[h264 @ 0x555e8b189840] left block unavailable for requested intra mode [h264 @ 0x555e8b189840] error while decoding MB 0 25, bytestream 5005 [h264 @ 0x555e8baea080] left block unavailable for requested intra mode [h264 @ 0x555e8baea080] error while decoding MB 0 39, bytestream 58008 [h264 @ 0x555e8b078000] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555e8b078000] error while decoding MB 0 18, bytestream 4205 [h264 @ 0x555e8b078000] reference picture missing during reorder [h", "last_exit_at": "2026-07-02T16:46:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 202.494, "last_started_at": "2026-07-02T16:42:46+09:00", "pid": 13159, "process": {"exists": false, "pid": 13159}, "restart_count": 53, "restart_count_per_hour": 9, "rtsp_timeout_count": 24, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 54}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 219.712, "sample_count": 1784, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 228180, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:46:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T16:46:08+09:00", "last_frame_read_age_sec": 0.141, "last_frame_read_at": "2026-07-02T16:46:08+09:00", "mjpeg": {"age_sec": 0.131, "exists": true, "mtime": "2026-07-02T16:46:08.487634+09:00", "mtime_epoch": 1782978368.4876337, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205032}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:46:08+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1392.281, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 79, "pushed_frames": 79, "read_frames": 78, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.636, "pushed": 2.636, "read": 2.602}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.709, "count": 78, "max_ms": 10.119, "p95_ms": 8.134}, "gstreamer_push": {"avg_ms": 1.067, "count": 79, "max_ms": 1.293, "p95_ms": 1.193}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.811, "count": 78, "max_ms": 73.671, "p95_ms": 64.391}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.59, "count": 78, "max_ms": 94.604, "p95_ms": 88.489}, "yolo11_640_fire_11class": {"avg_ms": 50.854, "count": 78, "max_ms": 74.623, "p95_ms": 68.613}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.125, "count": 78, "max_ms": 90.617, "p95_ms": 82.82}}, "truncated_count": 0}, "processing_total": {"avg_ms": 124.667, "count": 79, "max_ms": 172.084, "p95_ms": 156.059}, "read_interval": {"avg_ms": 384.169, "count": 78, "max_ms": 19453.062, "p95_ms": 172.928}, "triton_wait": {"avg_ms": 64.572, "count": 78, "max_ms": 94.616, "p95_ms": 89.227}, "visualize": {"avg_ms": 5.389, "count": 78, "max_ms": 6.767, "p95_ms": 6.485}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.972, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1901}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5504.133, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 41, "h264_decode_error_count": 84, "h264_decode_error_count_per_hour": 16, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:46:08+09:00", "last_error_summary": "[h264 @ 0x555e8b189840] left block unavailable for requested intra mode [h264 @ 0x555e8b189840] error while decoding MB 0 25, bytestream 5005 [h264 @ 0x555e8baea080] left block unavailable for requested intra mode [h264 @ 0x555e8baea080] error while decoding MB 0 39, bytestream 58008 [h264 @ 0x555e8b078000] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555e8b078000] error while decoding MB 0 18, bytestream 4205 [h264 @ 0x555e8b078000] reference picture missing during reorder [h", "last_exit_at": "2026-07-02T16:46:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 202.5, "last_started_at": "2026-07-02T16:42:46+09:00", "pid": 13159, "process": {"exists": false, "pid": 13159}, "restart_count": 53, "restart_count_per_hour": 9, "rtsp_timeout_count": 24, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 54}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 219.712, "sample_count": 1784, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 228180, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:46:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-02T16:46:08+09:00", "last_frame_read_age_sec": 0.146, "last_frame_read_at": "2026-07-02T16:46:08+09:00", "mjpeg": {"age_sec": 0.136, "exists": true, "mtime": "2026-07-02T16:46:08.487634+09:00", "mtime_epoch": 1782978368.4876337, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205032}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:46:08+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1392.281, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 79, "pushed_frames": 79, "read_frames": 78, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.635, "pushed": 2.635, "read": 2.602}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.709, "count": 78, "max_ms": 10.119, "p95_ms": 8.134}, "gstreamer_push": {"avg_ms": 1.067, "count": 79, "max_ms": 1.293, "p95_ms": 1.193}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.811, "count": 78, "max_ms": 73.671, "p95_ms": 64.391}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.59, "count": 78, "max_ms": 94.604, "p95_ms": 88.489}, "yolo11_640_fire_11class": {"avg_ms": 50.854, "count": 78, "max_ms": 74.623, "p95_ms": 68.613}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.125, "count": 78, "max_ms": 90.617, "p95_ms": 82.82}}, "truncated_count": 0}, "processing_total": {"avg_ms": 124.667, "count": 79, "max_ms": 172.084, "p95_ms": 156.059}, "read_interval": {"avg_ms": 384.169, "count": 78, "max_ms": 19453.062, "p95_ms": 172.928}, "triton_wait": {"avg_ms": 64.572, "count": 78, "max_ms": 94.616, "p95_ms": 89.227}, "visualize": {"avg_ms": 5.389, "count": 78, "max_ms": 6.767, "p95_ms": 6.485}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.978, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1901}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5618.062, "limit_mb": 8796093022207.996}, "detail": {"pid": 13478, "restart_count": 54, "start_count": 55}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 41, "h264_decode_error_count": 84, "h264_decode_error_count_per_hour": 16, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:46:08+09:00", "last_error_summary": "[h264 @ 0x555e8b189840] left block unavailable for requested intra mode [h264 @ 0x555e8b189840] error while decoding MB 0 25, bytestream 5005 [h264 @ 0x555e8baea080] left block unavailable for requested intra mode [h264 @ 0x555e8baea080] error while decoding MB 0 39, bytestream 58008 [h264 @ 0x555e8b078000] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x555e8b078000] error while decoding MB 0 18, bytestream 4205 [h264 @ 0x555e8b078000] reference picture missing during reorder [h", "last_exit_at": "2026-07-02T16:46:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T16:46:11+09:00", "pid": 13478, "process": {"exists": true, "fd_count": 5, "hwm_mb": 147.0, "name": "ffmpeg", "pid": 13478, "rss_mb": 147.0, "state": "S (sleeping)", "threads": 106}, "restart_count": 54, "restart_count_per_hour": 10, "rtsp_timeout_count": 24, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 55}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 218.133, "sample_count": 1763, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 228180, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:46:08+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.837, "last_frame_pushed_at": "2026-07-02T16:46:08+09:00", "last_frame_read_age_sec": 4.972, "last_frame_read_at": "2026-07-02T16:46:08+09:00", "mjpeg": {"age_sec": 4.814, "exists": true, "mtime": "2026-07-02T16:46:08.635637+09:00", "mtime_epoch": 1782978368.6356366, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 205032}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:46:13+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1392.281, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 42, "pushed_frames": 42, "read_frames": 41, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.4, "pushed": 1.4, "read": 1.367}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 7.119, "count": 41, "max_ms": 10.119, "p95_ms": 8.152}, "gstreamer_push": {"avg_ms": 1.049, "count": 42, "max_ms": 1.293, "p95_ms": 1.159}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.706, "count": 41, "max_ms": 69.074, "p95_ms": 62.139}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.045, "count": 41, "max_ms": 94.604, "p95_ms": 86.863}, "yolo11_640_fire_11class": {"avg_ms": 50.152, "count": 41, "max_ms": 70.936, "p95_ms": 64.365}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.389, "count": 41, "max_ms": 84.544, "p95_ms": 78.449}}, "truncated_count": 0}, "processing_total": {"avg_ms": 130.096, "count": 42, "max_ms": 172.084, "p95_ms": 163.137}, "read_interval": {"avg_ms": 612.796, "count": 41, "max_ms": 19453.062, "p95_ms": 182.238}, "triton_wait": {"avg_ms": 64.097, "count": 41, "max_ms": 94.616, "p95_ms": 86.874}, "visualize": {"avg_ms": 5.739, "count": 41, "max_ms": 6.767, "p95_ms": 6.499}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.995, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 1925}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5430.176, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x560dbd034100] error while decoding MB 15 26, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 33, "h264_decode_error_count": 34, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:50:54+09:00", "last_error_summary": "[h264 @ 0x560dbd034100] error while decoding MB 15 26, bytestream -5", "last_exit_at": "2026-07-02T16:45:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 342.699, "last_started_at": "2026-07-02T16:45:12+09:00", "pid": 13326, "process": {"exists": false, "pid": 13326}, "restart_count": 32, "restart_count_per_hour": 2, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 33}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 256.398, "sample_count": 1444, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 231038, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:50:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.074, "last_frame_pushed_at": "2026-07-02T16:50:54+09:00", "last_frame_read_age_sec": 0.063, "last_frame_read_at": "2026-07-02T16:50:54+09:00", "mjpeg": {"age_sec": 0.058, "exists": true, "mtime": "2026-07-02T16:50:54.732678+09:00", "mtime_epoch": 1782978654.7326777, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 256702}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T16:50:54+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1459.445, "name": "celery", "pid": 267, "rss_mb": 1220.566, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 139, "pushed_frames": 139, "read_frames": 139, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.663, "pushed": 4.663, "read": 4.663}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 15.61, "count": 139, "max_ms": 20.449, "p95_ms": 18.854}, "gstreamer_push": {"avg_ms": 0.873, "count": 139, "max_ms": 1.167, "p95_ms": 0.947}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 53.034, "count": 139, "max_ms": 72.264, "p95_ms": 61.633}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.607, "count": 139, "max_ms": 94.03, "p95_ms": 84.707}, "yolo11_640_fire_11class": {"avg_ms": 46.25, "count": 139, "max_ms": 65.291, "p95_ms": 54.943}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.953, "count": 139, "max_ms": 87.591, "p95_ms": 74.763}}, "truncated_count": 0}, "processing_total": {"avg_ms": 205.597, "count": 139, "max_ms": 259.927, "p95_ms": 239.237}, "read_interval": {"avg_ms": 215.535, "count": 139, "max_ms": 269.294, "p95_ms": 249.74}, "triton_wait": {"avg_ms": 59.614, "count": 139, "max_ms": 94.042, "p95_ms": 84.946}, "visualize": {"avg_ms": 11.618, "count": 139, "max_ms": 16.085, "p95_ms": 13.5}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.809, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 1925}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5430.035, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T16:50:54+09:00", "last_error_summary": "[h264 @ 0x560dbd034100] error while decoding MB 15 26, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 33, "h264_decode_error_count": 34, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:50:54+09:00", "last_error_summary": "[h264 @ 0x560dbd034100] error while decoding MB 15 26, bytestream -5", "last_exit_at": "2026-07-02T16:50:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 342.728, "last_started_at": "2026-07-02T16:45:12+09:00", "pid": 13326, "process": {"exists": false, "pid": 13326}, "restart_count": 32, "restart_count_per_hour": 2, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 33}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 256.398, "sample_count": 1445, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 231039, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:50:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T16:50:54+09:00", "last_frame_read_age_sec": 0.091, "last_frame_read_at": "2026-07-02T16:50:54+09:00", "mjpeg": {"age_sec": 0.087, "exists": true, "mtime": "2026-07-02T16:50:54.732678+09:00", "mtime_epoch": 1782978654.7326777, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 256702}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T16:50:54+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1459.445, "name": "celery", "pid": 267, "rss_mb": 1220.566, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 140, "pushed_frames": 140, "read_frames": 139, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.692, "pushed": 4.692, "read": 4.659}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 15.55, "count": 140, "max_ms": 20.449, "p95_ms": 18.783}, "gstreamer_push": {"avg_ms": 0.875, "count": 140, "max_ms": 1.232, "p95_ms": 0.947}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.985, "count": 140, "max_ms": 72.264, "p95_ms": 61.253}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.664, "count": 140, "max_ms": 94.03, "p95_ms": 82.392}, "yolo11_640_fire_11class": {"avg_ms": 46.258, "count": 140, "max_ms": 65.291, "p95_ms": 54.567}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.891, "count": 140, "max_ms": 87.591, "p95_ms": 70.303}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.725, "count": 140, "max_ms": 259.927, "p95_ms": 238.499}, "read_interval": {"avg_ms": 215.535, "count": 139, "max_ms": 269.294, "p95_ms": 249.74}, "triton_wait": {"avg_ms": 59.664, "count": 140, "max_ms": 94.042, "p95_ms": 84.72}, "visualize": {"avg_ms": 11.566, "count": 140, "max_ms": 16.085, "p95_ms": 13.468}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.837, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 1925}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5429.867, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 33, "h264_decode_error_count": 34, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:50:54+09:00", "last_error_summary": "[h264 @ 0x560dbd034100] error while decoding MB 15 26, bytestream -5", "last_exit_at": "2026-07-02T16:50:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 342.735, "last_started_at": "2026-07-02T16:45:12+09:00", "pid": 13326, "process": {"exists": false, "pid": 13326}, "restart_count": 32, "restart_count_per_hour": 2, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 33}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 256.398, "sample_count": 1445, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 231039, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:50:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-02T16:50:54+09:00", "last_frame_read_age_sec": 0.098, "last_frame_read_at": "2026-07-02T16:50:54+09:00", "mjpeg": {"age_sec": 0.094, "exists": true, "mtime": "2026-07-02T16:50:54.732678+09:00", "mtime_epoch": 1782978654.7326777, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 256702}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T16:50:54+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1459.445, "name": "celery", "pid": 267, "rss_mb": 1220.566, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 140, "pushed_frames": 140, "read_frames": 139, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.691, "pushed": 4.691, "read": 4.658}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 15.55, "count": 140, "max_ms": 20.449, "p95_ms": 18.783}, "gstreamer_push": {"avg_ms": 0.875, "count": 140, "max_ms": 1.232, "p95_ms": 0.947}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.985, "count": 140, "max_ms": 72.264, "p95_ms": 61.253}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.664, "count": 140, "max_ms": 94.03, "p95_ms": 82.392}, "yolo11_640_fire_11class": {"avg_ms": 46.258, "count": 140, "max_ms": 65.291, "p95_ms": 54.567}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.891, "count": 140, "max_ms": 87.591, "p95_ms": 70.303}}, "truncated_count": 0}, "processing_total": {"avg_ms": 204.725, "count": 140, "max_ms": 259.927, "p95_ms": 238.499}, "read_interval": {"avg_ms": 215.535, "count": 139, "max_ms": 269.294, "p95_ms": 249.74}, "triton_wait": {"avg_ms": 59.664, "count": 140, "max_ms": 94.042, "p95_ms": 84.72}, "visualize": {"avg_ms": 11.566, "count": 140, "max_ms": 16.085, "p95_ms": 13.468}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.844, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 1925}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5457.117, "limit_mb": 8796093022207.996}, "detail": {"pid": 13706, "restart_count": 33, "start_count": 34}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 33, "h264_decode_error_count": 34, "h264_decode_error_count_per_hour": 4, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:50:54+09:00", "last_error_summary": "[h264 @ 0x560dbd034100] error while decoding MB 15 26, bytestream -5", "last_exit_at": "2026-07-02T16:50:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T16:50:57+09:00", "pid": 13706, "process": {"exists": true, "fd_count": 4, "hwm_mb": 58.59, "name": "ffmpeg", "pid": 13706, "rss_mb": 52.703, "state": "R (running)", "threads": 3}, "restart_count": 33, "restart_count_per_hour": 3, "rtsp_timeout_count": 46, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 34}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 256.703, "sample_count": 1403, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 231039, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:50:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 4.439, "last_frame_pushed_at": "2026-07-02T16:50:54+09:00", "last_frame_read_age_sec": 4.525, "last_frame_read_at": "2026-07-02T16:50:54+09:00", "mjpeg": {"age_sec": 4.424, "exists": true, "mtime": "2026-07-02T16:50:54.828679+09:00", "mtime_epoch": 1782978654.8286793, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 255914}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T16:50:59+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1459.445, "name": "celery", "pid": 267, "rss_mb": 1220.566, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 121, "pushed_frames": 121, "read_frames": 120, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 4.036, "pushed": 4.036, "read": 4.002}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "frame_processing_p95_backlog_risk", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 15.462, "count": 120, "max_ms": 20.449, "p95_ms": 18.854}, "gstreamer_push": {"avg_ms": 0.878, "count": 121, "max_ms": 1.232, "p95_ms": 1.024}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.857, "count": 120, "max_ms": 72.264, "p95_ms": 60.716}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.782, "count": 120, "max_ms": 94.03, "p95_ms": 84.707}, "yolo11_640_fire_11class": {"avg_ms": 46.499, "count": 120, "max_ms": 65.291, "p95_ms": 54.567}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 55.829, "count": 120, "max_ms": 87.591, "p95_ms": 70.303}}, "truncated_count": 0}, "processing_total": {"avg_ms": 202.949, "count": 121, "max_ms": 259.927, "p95_ms": 239.237}, "read_interval": {"avg_ms": 213.861, "count": 120, "max_ms": 269.294, "p95_ms": 249.74}, "triton_wait": {"avg_ms": 59.564, "count": 120, "max_ms": 94.042, "p95_ms": 84.946}, "visualize": {"avg_ms": 11.42, "count": 120, "max_ms": 16.085, "p95_ms": 13.5}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.982, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1919}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5361.277, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x557db53e7ac0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557db53e7ac0] error while decoding MB 0 43, bytestream 6766 [h264 @ 0x557db5fd0740] cabac decode of qscale diff failed at 51 44 [h264 @ 0x557db5fd0740] error while decoding MB 51 44, bytestream 4192 [in#0/rtsp @ 0x557db53a5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557db53a5e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 42, "h264_decode_error_count": 87, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:51:00+09:00", "last_error_summary": "[h264 @ 0x557db53e7ac0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557db53e7ac0] error while decoding MB 0 43, bytestream 6766 [h264 @ 0x557db5fd0740] cabac decode of qscale diff failed at 51 44 [h264 @ 0x557db5fd0740] error while decoding MB 51 44, bytestream 4192 [in#0/rtsp @ 0x557db53a5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557db53a5e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T16:46:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 289.062, "last_started_at": "2026-07-02T16:46:11+09:00", "pid": 13478, "process": {"exists": false, "pid": 13478}, "restart_count": 54, "restart_count_per_hour": 10, "rtsp_timeout_count": 26, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 55}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 165.998, "sample_count": 2137, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 230287, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:51:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.049, "last_frame_pushed_at": "2026-07-02T16:51:00+09:00", "last_frame_read_age_sec": 0.038, "last_frame_read_at": "2026-07-02T16:51:00+09:00", "mjpeg": {"age_sec": 0.046, "exists": true, "mtime": "2026-07-02T16:51:00.464777+09:00", "mtime_epoch": 1782978660.4647772, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175065}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:51:00+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1393.312, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 72, "pushed_frames": 72, "read_frames": 72, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.405, "pushed": 2.405, "read": 2.405}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 7.769, "count": 72, "max_ms": 9.843, "p95_ms": 8.904}, "gstreamer_push": {"avg_ms": 0.958, "count": 72, "max_ms": 1.394, "p95_ms": 1.274}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.868, "count": 72, "max_ms": 70.95, "p95_ms": 66.274}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.005, "count": 72, "max_ms": 100.049, "p95_ms": 88.114}, "yolo11_640_fire_11class": {"avg_ms": 48.101, "count": 72, "max_ms": 65.289, "p95_ms": 64.418}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.645, "count": 72, "max_ms": 87.655, "p95_ms": 81.333}}, "truncated_count": 0}, "processing_total": {"avg_ms": 150.466, "count": 72, "max_ms": 195.86, "p95_ms": 187.918}, "read_interval": {"avg_ms": 416.232, "count": 72, "max_ms": 10132.954, "p95_ms": 202.324}, "triton_wait": {"avg_ms": 62.944, "count": 72, "max_ms": 100.061, "p95_ms": 88.129}, "visualize": {"avg_ms": 6.707, "count": 72, "max_ms": 8.588, "p95_ms": 7.895}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.938, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1919}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5360.102, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T16:51:00+09:00", "last_error_summary": "[h264 @ 0x557db53e7ac0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557db53e7ac0] error while decoding MB 0 43, bytestream 6766 [h264 @ 0x557db5fd0740] cabac decode of qscale diff failed at 51 44 [h264 @ 0x557db5fd0740] error while decoding MB 51 44, bytestream 4192 [in#0/rtsp @ 0x557db53a5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557db53a5e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 42, "h264_decode_error_count": 87, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:51:00+09:00", "last_error_summary": "[h264 @ 0x557db53e7ac0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557db53e7ac0] error while decoding MB 0 43, bytestream 6766 [h264 @ 0x557db5fd0740] cabac decode of qscale diff failed at 51 44 [h264 @ 0x557db5fd0740] error while decoding MB 51 44, bytestream 4192 [in#0/rtsp @ 0x557db53a5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557db53a5e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T16:51:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 289.22, "last_started_at": "2026-07-02T16:46:11+09:00", "pid": 13478, "process": {"exists": false, "pid": 13478}, "restart_count": 54, "restart_count_per_hour": 10, "rtsp_timeout_count": 26, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 55}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 166.085, "sample_count": 2138, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 230288, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:51:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T16:51:00+09:00", "last_frame_read_age_sec": 0.196, "last_frame_read_at": "2026-07-02T16:51:00+09:00", "mjpeg": {"age_sec": 0.204, "exists": true, "mtime": "2026-07-02T16:51:00.464777+09:00", "mtime_epoch": 1782978660.4647772, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175065}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:51:00+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1393.312, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 71, "pushed_frames": 71, "read_frames": 70, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.373, "pushed": 2.373, "read": 2.339}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 7.863, "count": 71, "max_ms": 9.843, "p95_ms": 8.904}, "gstreamer_push": {"avg_ms": 0.948, "count": 71, "max_ms": 1.378, "p95_ms": 1.218}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.998, "count": 71, "max_ms": 70.95, "p95_ms": 66.274}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.494, "count": 71, "max_ms": 100.049, "p95_ms": 88.114}, "yolo11_640_fire_11class": {"avg_ms": 48.51, "count": 71, "max_ms": 75.484, "p95_ms": 64.764}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.228, "count": 71, "max_ms": 89.404, "p95_ms": 82.73}}, "truncated_count": 0}, "processing_total": {"avg_ms": 153.39, "count": 71, "max_ms": 195.86, "p95_ms": 191.615}, "read_interval": {"avg_ms": 425.777, "count": 70, "max_ms": 10132.954, "p95_ms": 202.324}, "triton_wait": {"avg_ms": 63.576, "count": 71, "max_ms": 100.061, "p95_ms": 89.416}, "visualize": {"avg_ms": 6.794, "count": 71, "max_ms": 8.588, "p95_ms": 7.895}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.921, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1919}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5360.215, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 42, "h264_decode_error_count": 87, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:51:00+09:00", "last_error_summary": "[h264 @ 0x557db53e7ac0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557db53e7ac0] error while decoding MB 0 43, bytestream 6766 [h264 @ 0x557db5fd0740] cabac decode of qscale diff failed at 51 44 [h264 @ 0x557db5fd0740] error while decoding MB 51 44, bytestream 4192 [in#0/rtsp @ 0x557db53a5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557db53a5e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T16:51:00+09:00", "last_returncode": 0, "last_stable_duration_sec": 289.224, "last_started_at": "2026-07-02T16:46:11+09:00", "pid": 13478, "process": {"exists": false, "pid": 13478}, "restart_count": 54, "restart_count_per_hour": 10, "rtsp_timeout_count": 26, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 55}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 166.085, "sample_count": 2138, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 230288, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:51:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T16:51:00+09:00", "last_frame_read_age_sec": 0.2, "last_frame_read_at": "2026-07-02T16:51:00+09:00", "mjpeg": {"age_sec": 0.208, "exists": true, "mtime": "2026-07-02T16:51:00.464777+09:00", "mtime_epoch": 1782978660.4647772, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175065}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:51:00+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1393.312, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 71, "pushed_frames": 71, "read_frames": 70, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.373, "pushed": 2.373, "read": 2.339}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 7.863, "count": 71, "max_ms": 9.843, "p95_ms": 8.904}, "gstreamer_push": {"avg_ms": 0.948, "count": 71, "max_ms": 1.378, "p95_ms": 1.218}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.998, "count": 71, "max_ms": 70.95, "p95_ms": 66.274}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.494, "count": 71, "max_ms": 100.049, "p95_ms": 88.114}, "yolo11_640_fire_11class": {"avg_ms": 48.51, "count": 71, "max_ms": 75.484, "p95_ms": 64.764}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.228, "count": 71, "max_ms": 89.404, "p95_ms": 82.73}}, "truncated_count": 0}, "processing_total": {"avg_ms": 153.39, "count": 71, "max_ms": 195.86, "p95_ms": 191.615}, "read_interval": {"avg_ms": 425.777, "count": 70, "max_ms": 10132.954, "p95_ms": 202.324}, "triton_wait": {"avg_ms": 63.576, "count": 71, "max_ms": 100.061, "p95_ms": 89.416}, "visualize": {"avg_ms": 6.794, "count": 71, "max_ms": 8.588, "p95_ms": 7.895}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.925, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1919}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5504.453, "limit_mb": 8796093022207.996}, "detail": {"pid": 13858, "restart_count": 55, "start_count": 56}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 42, "h264_decode_error_count": 87, "h264_decode_error_count_per_hour": 19, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:51:00+09:00", "last_error_summary": "[h264 @ 0x557db53e7ac0] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x557db53e7ac0] error while decoding MB 0 43, bytestream 6766 [h264 @ 0x557db5fd0740] cabac decode of qscale diff failed at 51 44 [h264 @ 0x557db5fd0740] error while decoding MB 51 44, bytestream 4192 [in#0/rtsp @ 0x557db53a5e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557db53a5e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T16:51:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T16:51:03+09:00", "pid": 13858, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.707, "name": "ffmpeg", "pid": 13858, "rss_mb": 141.707, "state": "S (sleeping)", "threads": 106}, "restart_count": 55, "restart_count_per_hour": 11, "rtsp_timeout_count": 26, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 56}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 166.094, "sample_count": 2129, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 230288, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:51:00+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.71, "last_frame_pushed_at": "2026-07-02T16:51:00+09:00", "last_frame_read_age_sec": 4.903, "last_frame_read_at": "2026-07-02T16:51:00+09:00", "mjpeg": {"age_sec": 4.695, "exists": true, "mtime": "2026-07-02T16:51:00.680781+09:00", "mtime_epoch": 1782978660.680781, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 175543}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:51:05+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1393.312, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 39, "pushed_frames": 39, "read_frames": 39, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.3, "pushed": 1.3, "read": 1.3}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 8.307, "count": 39, "max_ms": 9.843, "p95_ms": 8.933}, "gstreamer_push": {"avg_ms": 0.918, "count": 39, "max_ms": 1.05, "p95_ms": 0.996}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.191, "count": 39, "max_ms": 70.95, "p95_ms": 66.81}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.795, "count": 39, "max_ms": 91.237, "p95_ms": 89.848}, "yolo11_640_fire_11class": {"avg_ms": 48.677, "count": 39, "max_ms": 75.484, "p95_ms": 65.289}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.832, "count": 39, "max_ms": 89.404, "p95_ms": 87.655}}, "truncated_count": 0}, "processing_total": {"avg_ms": 167.352, "count": 39, "max_ms": 195.86, "p95_ms": 192.736}, "read_interval": {"avg_ms": 647.679, "count": 39, "max_ms": 10132.954, "p95_ms": 8603.389}, "triton_wait": {"avg_ms": 65.326, "count": 39, "max_ms": 91.248, "p95_ms": 89.86}, "visualize": {"avg_ms": 7.412, "count": 39, "max_ms": 8.298, "p95_ms": 8.117}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1927}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5411.512, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55c463ac6080] cabac decode of qscale diff failed at 46 27 [h264 @ 0x55c463ac6080] error while decoding MB 46 27, bytestream 2329 [h264 @ 0x55c463a89b40] left block unavailable for requested intra mode [h264 @ 0x55c463a89b40] error while decoding MB 0 21, bytestream 407 [h264 @ 0x55c463a72240] error while decoding MB 4 42, bytestream -9", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 43, "h264_decode_error_count": 91, "h264_decode_error_count_per_hour": 23, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:53:44+09:00", "last_error_summary": "[h264 @ 0x55c463ac6080] cabac decode of qscale diff failed at 46 27 [h264 @ 0x55c463ac6080] error while decoding MB 46 27, bytestream 2329 [h264 @ 0x55c463a89b40] left block unavailable for requested intra mode [h264 @ 0x55c463a89b40] error while decoding MB 0 21, bytestream 407 [h264 @ 0x55c463a72240] error while decoding MB 4 42, bytestream -9", "last_exit_at": "2026-07-02T16:51:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 160.859, "last_started_at": "2026-07-02T16:51:03+09:00", "pid": 13858, "process": {"exists": false, "pid": 13858}, "restart_count": 55, "restart_count_per_hour": 11, "rtsp_timeout_count": 26, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 56}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 184.774, "sample_count": 1818, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 231294, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:53:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.066, "last_frame_pushed_at": "2026-07-02T16:53:44+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-02T16:53:44+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-02T16:53:44.187602+09:00", "mtime_epoch": 1782978824.1876018, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200751}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:53:44+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1393.828, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 204, "pushed_frames": 204, "read_frames": 204, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.806, "pushed": 6.806, "read": 6.806}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.494, "count": 203, "max_ms": 12.176, "p95_ms": 9.542}, "gstreamer_push": {"avg_ms": 1.019, "count": 204, "max_ms": 1.254, "p95_ms": 1.184}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.311, "count": 203, "max_ms": 72.95, "p95_ms": 66.471}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.146, "count": 203, "max_ms": 99.715, "p95_ms": 84.731}, "yolo11_640_fire_11class": {"avg_ms": 50.254, "count": 203, "max_ms": 75.827, "p95_ms": 66.21}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.027, "count": 203, "max_ms": 90.805, "p95_ms": 82.09}}, "truncated_count": 0}, "processing_total": {"avg_ms": 136.211, "count": 204, "max_ms": 201.401, "p95_ms": 179.746}, "read_interval": {"avg_ms": 147.593, "count": 204, "max_ms": 210.544, "p95_ms": 189.556}, "triton_wait": {"avg_ms": 63.094, "count": 203, "max_ms": 99.728, "p95_ms": 85.931}, "visualize": {"avg_ms": 6.103, "count": 203, "max_ms": 9.706, "p95_ms": 7.899}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.972, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1927}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5411.262, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T16:53:44+09:00", "last_error_summary": "[h264 @ 0x55c463ac6080] cabac decode of qscale diff failed at 46 27 [h264 @ 0x55c463ac6080] error while decoding MB 46 27, bytestream 2329 [h264 @ 0x55c463a89b40] left block unavailable for requested intra mode [h264 @ 0x55c463a89b40] error while decoding MB 0 21, bytestream 407 [h264 @ 0x55c463a72240] error while decoding MB 4 42, bytestream -9", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 43, "h264_decode_error_count": 91, "h264_decode_error_count_per_hour": 23, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:53:44+09:00", "last_error_summary": "[h264 @ 0x55c463ac6080] cabac decode of qscale diff failed at 46 27 [h264 @ 0x55c463ac6080] error while decoding MB 46 27, bytestream 2329 [h264 @ 0x55c463a89b40] left block unavailable for requested intra mode [h264 @ 0x55c463a89b40] error while decoding MB 0 21, bytestream 407 [h264 @ 0x55c463a72240] error while decoding MB 4 42, bytestream -9", "last_exit_at": "2026-07-02T16:53:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 160.904, "last_started_at": "2026-07-02T16:51:03+09:00", "pid": 13858, "process": {"exists": false, "pid": 13858}, "restart_count": 55, "restart_count_per_hour": 11, "rtsp_timeout_count": 26, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 56}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 184.774, "sample_count": 1819, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 231295, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:53:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T16:53:44+09:00", "last_frame_read_age_sec": 0.097, "last_frame_read_at": "2026-07-02T16:53:44+09:00", "mjpeg": {"age_sec": 0.091, "exists": true, "mtime": "2026-07-02T16:53:44.187602+09:00", "mtime_epoch": 1782978824.1876018, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200751}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:53:44+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1393.828, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 204, "pushed_frames": 204, "read_frames": 203, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.843, "pushed": 6.843, "read": 6.81}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.484, "count": 204, "max_ms": 12.176, "p95_ms": 9.542}, "gstreamer_push": {"avg_ms": 1.019, "count": 204, "max_ms": 1.254, "p95_ms": 1.184}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.313, "count": 204, "max_ms": 72.95, "p95_ms": 66.471}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.124, "count": 204, "max_ms": 99.715, "p95_ms": 84.731}, "yolo11_640_fire_11class": {"avg_ms": 50.25, "count": 204, "max_ms": 75.827, "p95_ms": 66.21}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.026, "count": 204, "max_ms": 90.805, "p95_ms": 82.09}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.733, "count": 204, "max_ms": 201.401, "p95_ms": 177.623}, "read_interval": {"avg_ms": 147.337, "count": 203, "max_ms": 210.544, "p95_ms": 187.552}, "triton_wait": {"avg_ms": 63.078, "count": 204, "max_ms": 99.728, "p95_ms": 85.931}, "visualize": {"avg_ms": 6.087, "count": 204, "max_ms": 9.706, "p95_ms": 7.899}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.811, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1927}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5411.379, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 43, "h264_decode_error_count": 91, "h264_decode_error_count_per_hour": 23, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:53:44+09:00", "last_error_summary": "[h264 @ 0x55c463ac6080] cabac decode of qscale diff failed at 46 27 [h264 @ 0x55c463ac6080] error while decoding MB 46 27, bytestream 2329 [h264 @ 0x55c463a89b40] left block unavailable for requested intra mode [h264 @ 0x55c463a89b40] error while decoding MB 0 21, bytestream 407 [h264 @ 0x55c463a72240] error while decoding MB 4 42, bytestream -9", "last_exit_at": "2026-07-02T16:53:44+09:00", "last_returncode": 0, "last_stable_duration_sec": 160.908, "last_started_at": "2026-07-02T16:51:03+09:00", "pid": 13858, "process": {"exists": false, "pid": 13858}, "restart_count": 55, "restart_count_per_hour": 11, "rtsp_timeout_count": 26, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 56}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 184.774, "sample_count": 1819, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 231295, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:53:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-02T16:53:44+09:00", "last_frame_read_age_sec": 0.101, "last_frame_read_at": "2026-07-02T16:53:44+09:00", "mjpeg": {"age_sec": 0.095, "exists": true, "mtime": "2026-07-02T16:53:44.187602+09:00", "mtime_epoch": 1782978824.1876018, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200751}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:53:44+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1393.828, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 204, "pushed_frames": 204, "read_frames": 203, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.842, "pushed": 6.842, "read": 6.809}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.484, "count": 204, "max_ms": 12.176, "p95_ms": 9.542}, "gstreamer_push": {"avg_ms": 1.019, "count": 204, "max_ms": 1.254, "p95_ms": 1.184}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.313, "count": 204, "max_ms": 72.95, "p95_ms": 66.471}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.124, "count": 204, "max_ms": 99.715, "p95_ms": 84.731}, "yolo11_640_fire_11class": {"avg_ms": 50.25, "count": 204, "max_ms": 75.827, "p95_ms": 66.21}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.026, "count": 204, "max_ms": 90.805, "p95_ms": 82.09}}, "truncated_count": 0}, "processing_total": {"avg_ms": 135.733, "count": 204, "max_ms": 201.401, "p95_ms": 177.623}, "read_interval": {"avg_ms": 147.337, "count": 203, "max_ms": 210.544, "p95_ms": 187.552}, "triton_wait": {"avg_ms": 63.078, "count": 204, "max_ms": 99.728, "p95_ms": 85.931}, "visualize": {"avg_ms": 6.087, "count": 204, "max_ms": 9.706, "p95_ms": 7.899}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.815, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1927}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5517.035, "limit_mb": 8796093022207.996}, "detail": {"pid": 13965, "restart_count": 56, "start_count": 57}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 43, "h264_decode_error_count": 91, "h264_decode_error_count_per_hour": 23, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:53:44+09:00", "last_error_summary": "[h264 @ 0x55c463ac6080] cabac decode of qscale diff failed at 46 27 [h264 @ 0x55c463ac6080] error while decoding MB 46 27, bytestream 2329 [h264 @ 0x55c463a89b40] left block unavailable for requested intra mode [h264 @ 0x55c463a89b40] error while decoding MB 0 21, bytestream 407 [h264 @ 0x55c463a72240] error while decoding MB 4 42, bytestream -9", "last_exit_at": "2026-07-02T16:53:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T16:53:46+09:00", "pid": 13965, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.648, "name": "ffmpeg", "pid": 13965, "rss_mb": 136.648, "state": "S (sleeping)", "threads": 106}, "restart_count": 56, "restart_count_per_hour": 12, "rtsp_timeout_count": 26, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 57}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 184.788, "sample_count": 1788, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 231295, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:53:44+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.449, "last_frame_pushed_at": "2026-07-02T16:53:44+09:00", "last_frame_read_age_sec": 4.543, "last_frame_read_at": "2026-07-02T16:53:44+09:00", "mjpeg": {"age_sec": 4.429, "exists": true, "mtime": "2026-07-02T16:53:44.295604+09:00", "mtime_epoch": 1782978824.2956035, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 201411}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:53:48+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1393.828, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 179, "pushed_frames": 179, "read_frames": 178, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.973, "pushed": 5.973, "read": 5.94}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.295, "count": 178, "max_ms": 11.077, "p95_ms": 9.237}, "gstreamer_push": {"avg_ms": 1.035, "count": 179, "max_ms": 1.254, "p95_ms": 1.197}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.635, "count": 178, "max_ms": 72.272, "p95_ms": 66.554}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.421, "count": 178, "max_ms": 99.715, "p95_ms": 86.146}, "yolo11_640_fire_11class": {"avg_ms": 50.529, "count": 178, "max_ms": 75.827, "p95_ms": 66.528}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.424, "count": 178, "max_ms": 90.613, "p95_ms": 82.38}}, "truncated_count": 0}, "processing_total": {"avg_ms": 131.939, "count": 179, "max_ms": 201.401, "p95_ms": 173.496}, "read_interval": {"avg_ms": 143.689, "count": 178, "max_ms": 210.544, "p95_ms": 184.155}, "triton_wait": {"avg_ms": 63.411, "count": 178, "max_ms": 99.728, "p95_ms": 86.159}, "visualize": {"avg_ms": 5.875, "count": 178, "max_ms": 9.006, "p95_ms": 7.871}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.966, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1938}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5414.73, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55b4d33ff080] error while decoding MB 42 20, bytestream -12", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 44, "h264_decode_error_count": 92, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:56:58+09:00", "last_error_summary": "[h264 @ 0x55b4d33ff080] error while decoding MB 42 20, bytestream -12", "last_exit_at": "2026-07-02T16:53:46+09:00", "last_returncode": 0, "last_stable_duration_sec": 192.005, "last_started_at": "2026-07-02T16:53:46+09:00", "pid": 13965, "process": {"exists": false, "pid": 13965}, "restart_count": 56, "restart_count_per_hour": 12, "rtsp_timeout_count": 26, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 57}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 183.463, "sample_count": 2021, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 232679, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:56:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.056, "last_frame_pushed_at": "2026-07-02T16:56:58+09:00", "last_frame_read_age_sec": 0.046, "last_frame_read_at": "2026-07-02T16:56:58+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-02T16:56:58.682921+09:00", "mtime_epoch": 1782979018.6829212, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200737}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:56:58+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1394.344, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 227, "pushed_frames": 227, "read_frames": 227, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.585, "pushed": 7.585, "read": 7.585}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.639, "count": 227, "max_ms": 11.193, "p95_ms": 8.069}, "gstreamer_push": {"avg_ms": 1.09, "count": 227, "max_ms": 1.482, "p95_ms": 1.309}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.977, "count": 227, "max_ms": 72.612, "p95_ms": 66.601}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.435, "count": 227, "max_ms": 100.878, "p95_ms": 88.263}, "yolo11_640_fire_11class": {"avg_ms": 52.569, "count": 227, "max_ms": 74.434, "p95_ms": 69.582}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.328, "count": 227, "max_ms": 90.99, "p95_ms": 82.725}}, "truncated_count": 0}, "processing_total": {"avg_ms": 120.022, "count": 227, "max_ms": 172.667, "p95_ms": 150.208}, "read_interval": {"avg_ms": 132.395, "count": 227, "max_ms": 183.874, "p95_ms": 161.674}, "triton_wait": {"avg_ms": 65.802, "count": 227, "max_ms": 100.892, "p95_ms": 88.92}, "visualize": {"avg_ms": 5.233, "count": 227, "max_ms": 8.42, "p95_ms": 6.388}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.927, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1939}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5414.0, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T16:56:58+09:00", "last_error_summary": "[h264 @ 0x55b4d33ff080] error while decoding MB 42 20, bytestream -12", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 44, "h264_decode_error_count": 92, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:56:58+09:00", "last_error_summary": "[h264 @ 0x55b4d33ff080] error while decoding MB 42 20, bytestream -12", "last_exit_at": "2026-07-02T16:56:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 192.08, "last_started_at": "2026-07-02T16:53:46+09:00", "pid": 13965, "process": {"exists": false, "pid": 13965}, "restart_count": 56, "restart_count_per_hour": 12, "rtsp_timeout_count": 26, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 57}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 183.463, "sample_count": 2022, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 232680, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:56:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T16:56:58+09:00", "last_frame_read_age_sec": 0.121, "last_frame_read_at": "2026-07-02T16:56:58+09:00", "mjpeg": {"age_sec": 0.12, "exists": true, "mtime": "2026-07-02T16:56:58.682921+09:00", "mtime_epoch": 1782979018.6829212, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200737}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:56:58+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1394.344, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 227, "pushed_frames": 227, "read_frames": 227, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.569, "pushed": 7.569, "read": 7.569}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.637, "count": 227, "max_ms": 11.193, "p95_ms": 8.069}, "gstreamer_push": {"avg_ms": 1.089, "count": 227, "max_ms": 1.482, "p95_ms": 1.309}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.913, "count": 227, "max_ms": 72.612, "p95_ms": 66.601}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.306, "count": 227, "max_ms": 100.878, "p95_ms": 87.941}, "yolo11_640_fire_11class": {"avg_ms": 52.474, "count": 227, "max_ms": 74.434, "p95_ms": 69.149}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.168, "count": 227, "max_ms": 90.573, "p95_ms": 82.613}}, "truncated_count": 0}, "processing_total": {"avg_ms": 119.768, "count": 227, "max_ms": 172.667, "p95_ms": 149.494}, "read_interval": {"avg_ms": 132.395, "count": 227, "max_ms": 183.874, "p95_ms": 161.674}, "triton_wait": {"avg_ms": 65.67, "count": 227, "max_ms": 100.892, "p95_ms": 88.289}, "visualize": {"avg_ms": 5.226, "count": 227, "max_ms": 8.42, "p95_ms": 6.366}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.991, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1939}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5413.91, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 44, "h264_decode_error_count": 92, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:56:58+09:00", "last_error_summary": "[h264 @ 0x55b4d33ff080] error while decoding MB 42 20, bytestream -12", "last_exit_at": "2026-07-02T16:56:58+09:00", "last_returncode": 0, "last_stable_duration_sec": 192.085, "last_started_at": "2026-07-02T16:53:46+09:00", "pid": 13965, "process": {"exists": false, "pid": 13965}, "restart_count": 56, "restart_count_per_hour": 12, "rtsp_timeout_count": 26, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 57}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 183.463, "sample_count": 2022, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 232680, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:56:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-02T16:56:58+09:00", "last_frame_read_age_sec": 0.126, "last_frame_read_at": "2026-07-02T16:56:58+09:00", "mjpeg": {"age_sec": 0.126, "exists": true, "mtime": "2026-07-02T16:56:58.682921+09:00", "mtime_epoch": 1782979018.6829212, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200737}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:56:58+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1394.344, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 227, "pushed_frames": 227, "read_frames": 227, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.568, "pushed": 7.568, "read": 7.568}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.637, "count": 227, "max_ms": 11.193, "p95_ms": 8.069}, "gstreamer_push": {"avg_ms": 1.089, "count": 227, "max_ms": 1.482, "p95_ms": 1.309}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.913, "count": 227, "max_ms": 72.612, "p95_ms": 66.601}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.306, "count": 227, "max_ms": 100.878, "p95_ms": 87.941}, "yolo11_640_fire_11class": {"avg_ms": 52.474, "count": 227, "max_ms": 74.434, "p95_ms": 69.149}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.168, "count": 227, "max_ms": 90.573, "p95_ms": 82.613}}, "truncated_count": 0}, "processing_total": {"avg_ms": 119.768, "count": 227, "max_ms": 172.667, "p95_ms": 149.494}, "read_interval": {"avg_ms": 132.395, "count": 227, "max_ms": 183.874, "p95_ms": 161.674}, "triton_wait": {"avg_ms": 65.67, "count": 227, "max_ms": 100.892, "p95_ms": 88.289}, "visualize": {"avg_ms": 5.226, "count": 227, "max_ms": 8.42, "p95_ms": 6.366}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1939}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5522.449, "limit_mb": 8796093022207.996}, "detail": {"pid": 14072, "restart_count": 57, "start_count": 58}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 44, "h264_decode_error_count": 92, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T16:56:58+09:00", "last_error_summary": "[h264 @ 0x55b4d33ff080] error while decoding MB 42 20, bytestream -12", "last_exit_at": "2026-07-02T16:57:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T16:57:01+09:00", "pid": 14072, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.871, "name": "ffmpeg", "pid": 14072, "rss_mb": 138.871, "state": "S (sleeping)", "threads": 106}, "restart_count": 57, "restart_count_per_hour": 13, "rtsp_timeout_count": 26, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 58}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 183.122, "sample_count": 1993, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 232680, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T16:56:58+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.995, "last_frame_pushed_at": "2026-07-02T16:56:58+09:00", "last_frame_read_age_sec": 5.111, "last_frame_read_at": "2026-07-02T16:56:58+09:00", "mjpeg": {"age_sec": 4.97, "exists": true, "mtime": "2026-07-02T16:56:58.822924+09:00", "mtime_epoch": 1782979018.8229237, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 200807}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T16:57:03+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1394.344, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 190, "pushed_frames": 190, "read_frames": 189, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.356, "pushed": 6.356, "read": 6.323}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.603, "count": 190, "max_ms": 11.193, "p95_ms": 8.229}, "gstreamer_push": {"avg_ms": 1.092, "count": 190, "max_ms": 1.482, "p95_ms": 1.34}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.167, "count": 190, "max_ms": 72.612, "p95_ms": 66.601}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.58, "count": 190, "max_ms": 100.878, "p95_ms": 88.278}, "yolo11_640_fire_11class": {"avg_ms": 52.601, "count": 190, "max_ms": 73.463, "p95_ms": 69.149}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.365, "count": 190, "max_ms": 90.573, "p95_ms": 82.613}}, "truncated_count": 0}, "processing_total": {"avg_ms": 119.341, "count": 190, "max_ms": 172.667, "p95_ms": 153.862}, "read_interval": {"avg_ms": 131.789, "count": 189, "max_ms": 183.812, "p95_ms": 166.231}, "triton_wait": {"avg_ms": 65.94, "count": 190, "max_ms": 100.892, "p95_ms": 90.586}, "visualize": {"avg_ms": 5.203, "count": 190, "max_ms": 7.709, "p95_ms": 6.335}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.891, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1950}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5417.035, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 2, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55f830ca2ec0] error while decoding MB 30 18, bytestream -24 [h264 @ 0x55f830da34c0] error while decoding MB 38 30, bytestream -38 [h264 @ 0x55f830cf6540] reference picture missing during reorder [h264 @ 0x55f830cf6540] Missing reference picture, default is 65568 [h264 @ 0x55f830ca2ec0] mmco: unref short failure [h264 @ 0x55f830ca2ec0] illegal short term buffer state detected [in#0/rtsp @ 0x55f830c72e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f830c72e00] Error retr", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 45, "h264_decode_error_count": 94, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T17:00:32+09:00", "last_error_summary": "[h264 @ 0x55f830ca2ec0] error while decoding MB 30 18, bytestream -24 [h264 @ 0x55f830da34c0] error while decoding MB 38 30, bytestream -38 [h264 @ 0x55f830cf6540] reference picture missing during reorder [h264 @ 0x55f830cf6540] Missing reference picture, default is 65568 [h264 @ 0x55f830ca2ec0] mmco: unref short failure [h264 @ 0x55f830ca2ec0] illegal short term buffer state detected [in#0/rtsp @ 0x55f830c72e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f830c72e00] Error retr", "last_exit_at": "2026-07-02T16:57:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 210.608, "last_started_at": "2026-07-02T16:57:01+09:00", "pid": 14072, "process": {"exists": false, "pid": 14072}, "restart_count": 57, "restart_count_per_hour": 13, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 58}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 168.309, "sample_count": 1965, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 234048, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T17:00:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.067, "last_frame_pushed_at": "2026-07-02T17:00:32+09:00", "last_frame_read_age_sec": 0.058, "last_frame_read_at": "2026-07-02T17:00:32+09:00", "mjpeg": {"age_sec": 0.05, "exists": true, "mtime": "2026-07-02T17:00:32.350537+09:00", "mtime_epoch": 1782979232.3505366, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204163}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T17:00:32+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1395.117, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 85, "pushed_frames": 85, "read_frames": 85, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.842, "pushed": 2.842, "read": 2.842}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.771, "count": 85, "max_ms": 9.854, "p95_ms": 8.316}, "gstreamer_push": {"avg_ms": 1.084, "count": 85, "max_ms": 1.228, "p95_ms": 1.199}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.436, "count": 85, "max_ms": 83.481, "p95_ms": 65.095}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.025, "count": 85, "max_ms": 89.28, "p95_ms": 86.716}, "yolo11_640_fire_11class": {"avg_ms": 53.846, "count": 85, "max_ms": 85.633, "p95_ms": 69.304}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.947, "count": 85, "max_ms": 87.661, "p95_ms": 81.873}}, "truncated_count": 0}, "processing_total": {"avg_ms": 122.456, "count": 85, "max_ms": 165.747, "p95_ms": 142.718}, "read_interval": {"avg_ms": 352.754, "count": 85, "max_ms": 10130.171, "p95_ms": 156.801}, "triton_wait": {"avg_ms": 66.01, "count": 85, "max_ms": 89.293, "p95_ms": 86.729}, "visualize": {"avg_ms": 5.285, "count": 85, "max_ms": 7.325, "p95_ms": 6.748}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.911, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1950}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5407.934, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T17:00:32+09:00", "last_error_summary": "[h264 @ 0x55f830ca2ec0] error while decoding MB 30 18, bytestream -24 [h264 @ 0x55f830da34c0] error while decoding MB 38 30, bytestream -38 [h264 @ 0x55f830cf6540] reference picture missing during reorder [h264 @ 0x55f830cf6540] Missing reference picture, default is 65568 [h264 @ 0x55f830ca2ec0] mmco: unref short failure [h264 @ 0x55f830ca2ec0] illegal short term buffer state detected [in#0/rtsp @ 0x55f830c72e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f830c72e00] Error retr", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 45, "h264_decode_error_count": 94, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T17:00:32+09:00", "last_error_summary": "[h264 @ 0x55f830ca2ec0] error while decoding MB 30 18, bytestream -24 [h264 @ 0x55f830da34c0] error while decoding MB 38 30, bytestream -38 [h264 @ 0x55f830cf6540] reference picture missing during reorder [h264 @ 0x55f830cf6540] Missing reference picture, default is 65568 [h264 @ 0x55f830ca2ec0] mmco: unref short failure [h264 @ 0x55f830ca2ec0] illegal short term buffer state detected [in#0/rtsp @ 0x55f830c72e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f830c72e00] Error retr", "last_exit_at": "2026-07-02T17:00:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 210.707, "last_started_at": "2026-07-02T16:57:01+09:00", "pid": 14072, "process": {"exists": false, "pid": 14072}, "restart_count": 57, "restart_count_per_hour": 13, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 58}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 168.309, "sample_count": 1965, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 234049, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T17:00:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.003, "last_frame_pushed_at": "2026-07-02T17:00:32+09:00", "last_frame_read_age_sec": 0.157, "last_frame_read_at": "2026-07-02T17:00:32+09:00", "mjpeg": {"age_sec": 0.149, "exists": true, "mtime": "2026-07-02T17:00:32.350537+09:00", "mtime_epoch": 1782979232.3505366, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204163}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T17:00:32+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1395.117, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 85, "pushed_frames": 85, "read_frames": 85, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.833, "pushed": 2.833, "read": 2.833}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.792, "count": 85, "max_ms": 9.854, "p95_ms": 8.316}, "gstreamer_push": {"avg_ms": 1.08, "count": 85, "max_ms": 1.228, "p95_ms": 1.199}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.4, "count": 85, "max_ms": 83.481, "p95_ms": 65.095}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.867, "count": 85, "max_ms": 89.28, "p95_ms": 86.716}, "yolo11_640_fire_11class": {"avg_ms": 53.853, "count": 85, "max_ms": 85.633, "p95_ms": 69.304}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.841, "count": 85, "max_ms": 87.661, "p95_ms": 81.873}}, "truncated_count": 0}, "processing_total": {"avg_ms": 122.702, "count": 85, "max_ms": 165.747, "p95_ms": 143.106}, "read_interval": {"avg_ms": 352.754, "count": 85, "max_ms": 10130.171, "p95_ms": 156.801}, "triton_wait": {"avg_ms": 65.878, "count": 85, "max_ms": 89.293, "p95_ms": 86.729}, "visualize": {"avg_ms": 5.301, "count": 85, "max_ms": 7.325, "p95_ms": 6.748}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1950}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5407.949, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 45, "h264_decode_error_count": 94, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T17:00:32+09:00", "last_error_summary": "[h264 @ 0x55f830ca2ec0] error while decoding MB 30 18, bytestream -24 [h264 @ 0x55f830da34c0] error while decoding MB 38 30, bytestream -38 [h264 @ 0x55f830cf6540] reference picture missing during reorder [h264 @ 0x55f830cf6540] Missing reference picture, default is 65568 [h264 @ 0x55f830ca2ec0] mmco: unref short failure [h264 @ 0x55f830ca2ec0] illegal short term buffer state detected [in#0/rtsp @ 0x55f830c72e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f830c72e00] Error retr", "last_exit_at": "2026-07-02T17:00:32+09:00", "last_returncode": 0, "last_stable_duration_sec": 210.711, "last_started_at": "2026-07-02T16:57:01+09:00", "pid": 14072, "process": {"exists": false, "pid": 14072}, "restart_count": 57, "restart_count_per_hour": 13, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 58}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 168.309, "sample_count": 1965, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 234049, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T17:00:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-02T17:00:32+09:00", "last_frame_read_age_sec": 0.161, "last_frame_read_at": "2026-07-02T17:00:32+09:00", "mjpeg": {"age_sec": 0.153, "exists": true, "mtime": "2026-07-02T17:00:32.350537+09:00", "mtime_epoch": 1782979232.3505366, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204163}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T17:00:32+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1395.117, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 85, "pushed_frames": 85, "read_frames": 84, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.846, "pushed": 2.846, "read": 2.813}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.792, "count": 85, "max_ms": 9.854, "p95_ms": 8.316}, "gstreamer_push": {"avg_ms": 1.08, "count": 85, "max_ms": 1.228, "p95_ms": 1.199}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.4, "count": 85, "max_ms": 83.481, "p95_ms": 65.095}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.867, "count": 85, "max_ms": 89.28, "p95_ms": 86.716}, "yolo11_640_fire_11class": {"avg_ms": 53.853, "count": 85, "max_ms": 85.633, "p95_ms": 69.304}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.841, "count": 85, "max_ms": 87.661, "p95_ms": 81.873}}, "truncated_count": 0}, "processing_total": {"avg_ms": 122.702, "count": 85, "max_ms": 165.747, "p95_ms": 143.106}, "read_interval": {"avg_ms": 355.255, "count": 84, "max_ms": 10130.171, "p95_ms": 156.801}, "triton_wait": {"avg_ms": 65.878, "count": 85, "max_ms": 89.293, "p95_ms": 86.729}, "visualize": {"avg_ms": 5.301, "count": 85, "max_ms": 7.325, "p95_ms": 6.748}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.865, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1950}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5524.477, "limit_mb": 8796093022207.996}, "detail": {"pid": 14179, "restart_count": 58, "start_count": 59}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 45, "h264_decode_error_count": 94, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T17:00:32+09:00", "last_error_summary": "[h264 @ 0x55f830ca2ec0] error while decoding MB 30 18, bytestream -24 [h264 @ 0x55f830da34c0] error while decoding MB 38 30, bytestream -38 [h264 @ 0x55f830cf6540] reference picture missing during reorder [h264 @ 0x55f830cf6540] Missing reference picture, default is 65568 [h264 @ 0x55f830ca2ec0] mmco: unref short failure [h264 @ 0x55f830ca2ec0] illegal short term buffer state detected [in#0/rtsp @ 0x55f830c72e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f830c72e00] Error retr", "last_exit_at": "2026-07-02T17:00:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T17:00:35+09:00", "pid": 14179, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.84, "name": "ffmpeg", "pid": 14179, "rss_mb": 136.84, "state": "S (sleeping)", "threads": 106}, "restart_count": 58, "restart_count_per_hour": 14, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 59}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 169.115, "sample_count": 1924, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 234049, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T17:00:32+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.803, "last_frame_pushed_at": "2026-07-02T17:00:32+09:00", "last_frame_read_age_sec": 4.956, "last_frame_read_at": "2026-07-02T17:00:32+09:00", "mjpeg": {"age_sec": 4.784, "exists": true, "mtime": "2026-07-02T17:00:32.514539+09:00", "mtime_epoch": 1782979232.5145395, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 204633}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T17:00:37+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1395.117, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 49, "pushed_frames": 49, "read_frames": 48, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.64, "pushed": 1.64, "read": 1.606}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.858, "count": 49, "max_ms": 9.854, "p95_ms": 8.316}, "gstreamer_push": {"avg_ms": 1.074, "count": 49, "max_ms": 1.228, "p95_ms": 1.199}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.831, "count": 49, "max_ms": 83.481, "p95_ms": 66.474}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.295, "count": 49, "max_ms": 89.28, "p95_ms": 86.716}, "yolo11_640_fire_11class": {"avg_ms": 52.899, "count": 49, "max_ms": 85.633, "p95_ms": 69.722}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.519, "count": 49, "max_ms": 87.661, "p95_ms": 81.873}}, "truncated_count": 0}, "processing_total": {"avg_ms": 123.809, "count": 49, "max_ms": 165.747, "p95_ms": 153.069}, "read_interval": {"avg_ms": 522.195, "count": 48, "max_ms": 10130.171, "p95_ms": 175.59}, "triton_wait": {"avg_ms": 64.547, "count": 49, "max_ms": 89.293, "p95_ms": 86.729}, "visualize": {"avg_ms": 5.434, "count": 49, "max_ms": 7.325, "p95_ms": 6.858}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.879, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1959}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5420.117, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T17:00:32+09:00", "last_error_summary": "[h264 @ 0x55f830ca2ec0] error while decoding MB 30 18, bytestream -24 [h264 @ 0x55f830da34c0] error while decoding MB 38 30, bytestream -38 [h264 @ 0x55f830cf6540] reference picture missing during reorder [h264 @ 0x55f830cf6540] Missing reference picture, default is 65568 [h264 @ 0x55f830ca2ec0] mmco: unref short failure [h264 @ 0x55f830ca2ec0] illegal short term buffer state detected [in#0/rtsp @ 0x55f830c72e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f830c72e00] Error retr", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 45, "h264_decode_error_count": 94, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T17:00:32+09:00", "last_error_summary": "[h264 @ 0x55f830ca2ec0] error while decoding MB 30 18, bytestream -24 [h264 @ 0x55f830da34c0] error while decoding MB 38 30, bytestream -38 [h264 @ 0x55f830cf6540] reference picture missing during reorder [h264 @ 0x55f830cf6540] Missing reference picture, default is 65568 [h264 @ 0x55f830ca2ec0] mmco: unref short failure [h264 @ 0x55f830ca2ec0] illegal short term buffer state detected [in#0/rtsp @ 0x55f830c72e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f830c72e00] Error retr", "last_exit_at": "2026-07-02T17:03:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.995, "last_started_at": "2026-07-02T17:00:35+09:00", "pid": 14179, "process": {"exists": false, "pid": 14179}, "restart_count": 58, "restart_count_per_hour": 14, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 59}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 170.984, "sample_count": 1894, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 235134, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T17:03:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T17:03:19+09:00", "last_frame_read_age_sec": 0.173, "last_frame_read_at": "2026-07-02T17:03:19+09:00", "mjpeg": {"age_sec": 0.167, "exists": true, "mtime": "2026-07-02T17:03:19.125343+09:00", "mtime_epoch": 1782979399.125343, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 167620}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T17:03:19+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1395.633, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 195, "pushed_frames": 196, "read_frames": 195, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.5, "pushed": 6.533, "read": 6.5}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.648, "count": 195, "max_ms": 11.838, "p95_ms": 9.695}, "gstreamer_push": {"avg_ms": 1.012, "count": 196, "max_ms": 1.714, "p95_ms": 1.198}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.171, "count": 195, "max_ms": 72.621, "p95_ms": 65.751}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.105, "count": 195, "max_ms": 105.462, "p95_ms": 86.55}, "yolo11_640_fire_11class": {"avg_ms": 49.939, "count": 195, "max_ms": 69.749, "p95_ms": 65.065}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.705, "count": 195, "max_ms": 91.802, "p95_ms": 83.299}}, "truncated_count": 0}, "processing_total": {"avg_ms": 142.362, "count": 195, "max_ms": 193.078, "p95_ms": 176.201}, "read_interval": {"avg_ms": 153.647, "count": 195, "max_ms": 203.436, "p95_ms": 186.507}, "triton_wait": {"avg_ms": 63.67, "count": 195, "max_ms": 105.474, "p95_ms": 86.86}, "visualize": {"avg_ms": 6.335, "count": 195, "max_ms": 8.948, "p95_ms": 7.943}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1959}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5420.109, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 45, "h264_decode_error_count": 94, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T17:00:32+09:00", "last_error_summary": "[h264 @ 0x55f830ca2ec0] error while decoding MB 30 18, bytestream -24 [h264 @ 0x55f830da34c0] error while decoding MB 38 30, bytestream -38 [h264 @ 0x55f830cf6540] reference picture missing during reorder [h264 @ 0x55f830cf6540] Missing reference picture, default is 65568 [h264 @ 0x55f830ca2ec0] mmco: unref short failure [h264 @ 0x55f830ca2ec0] illegal short term buffer state detected [in#0/rtsp @ 0x55f830c72e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f830c72e00] Error retr", "last_exit_at": "2026-07-02T17:03:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 163.999, "last_started_at": "2026-07-02T17:00:35+09:00", "pid": 14179, "process": {"exists": false, "pid": 14179}, "restart_count": 58, "restart_count_per_hour": 14, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": false, "start_count": 59}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 170.984, "sample_count": 1894, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 235134, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T17:03:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-02T17:03:19+09:00", "last_frame_read_age_sec": 0.177, "last_frame_read_at": "2026-07-02T17:03:19+09:00", "mjpeg": {"age_sec": 0.171, "exists": true, "mtime": "2026-07-02T17:03:19.125343+09:00", "mtime_epoch": 1782979399.125343, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 167620}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T17:03:19+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1395.633, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 195, "pushed_frames": 195, "read_frames": 195, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.502, "pushed": 6.502, "read": 6.502}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.648, "count": 195, "max_ms": 11.838, "p95_ms": 9.695}, "gstreamer_push": {"avg_ms": 1.011, "count": 195, "max_ms": 1.714, "p95_ms": 1.198}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.171, "count": 195, "max_ms": 72.621, "p95_ms": 65.751}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.105, "count": 195, "max_ms": 105.462, "p95_ms": 86.55}, "yolo11_640_fire_11class": {"avg_ms": 49.939, "count": 195, "max_ms": 69.749, "p95_ms": 65.065}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.705, "count": 195, "max_ms": 91.802, "p95_ms": 83.299}}, "truncated_count": 0}, "processing_total": {"avg_ms": 142.362, "count": 195, "max_ms": 193.078, "p95_ms": 176.201}, "read_interval": {"avg_ms": 153.647, "count": 195, "max_ms": 203.436, "p95_ms": 186.507}, "triton_wait": {"avg_ms": 63.67, "count": 195, "max_ms": 105.474, "p95_ms": 86.86}, "visualize": {"avg_ms": 6.335, "count": 195, "max_ms": 8.948, "p95_ms": 7.943}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.992, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1959}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5526.516, "limit_mb": 8796093022207.996}, "detail": {"pid": 14286, "restart_count": 59, "start_count": 60}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 45, "h264_decode_error_count": 94, "h264_decode_error_count_per_hour": 26, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T17:00:32+09:00", "last_error_summary": "[h264 @ 0x55f830ca2ec0] error while decoding MB 30 18, bytestream -24 [h264 @ 0x55f830da34c0] error while decoding MB 38 30, bytestream -38 [h264 @ 0x55f830cf6540] reference picture missing during reorder [h264 @ 0x55f830cf6540] Missing reference picture, default is 65568 [h264 @ 0x55f830ca2ec0] mmco: unref short failure [h264 @ 0x55f830ca2ec0] illegal short term buffer state detected [in#0/rtsp @ 0x55f830c72e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55f830c72e00] Error retr", "last_exit_at": "2026-07-02T17:03:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T17:03:21+09:00", "pid": 14286, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.672, "name": "ffmpeg", "pid": 14286, "rss_mb": 138.672, "state": "S (sleeping)", "threads": 106}, "restart_count": 59, "restart_count_per_hour": 15, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 10, "rtsp_tolerant_mode": false, "running": true, "start_count": 60}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 171.073, "sample_count": 1867, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 235134, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T17:03:19+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["rtsp_timeout_count_per_hour>=10", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.034, "last_frame_pushed_at": "2026-07-02T17:03:19+09:00", "last_frame_read_age_sec": 4.203, "last_frame_read_at": "2026-07-02T17:03:19+09:00", "mjpeg": {"age_sec": 4.018, "exists": true, "mtime": "2026-07-02T17:03:19.305346+09:00", "mtime_epoch": 1782979399.305346, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 168157}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T17:03:23+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1395.633, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "rtsp_timeout_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 171, "pushed_frames": 171, "read_frames": 170, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 5.72, "pushed": 5.72, "read": 5.687}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.573, "count": 171, "max_ms": 11.838, "p95_ms": 9.615}, "gstreamer_push": {"avg_ms": 1.022, "count": 171, "max_ms": 1.714, "p95_ms": 1.199}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.207, "count": 171, "max_ms": 72.621, "p95_ms": 65.694}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.271, "count": 171, "max_ms": 105.462, "p95_ms": 86.848}, "yolo11_640_fire_11class": {"avg_ms": 50.111, "count": 171, "max_ms": 69.749, "p95_ms": 65.301}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.0, "count": 171, "max_ms": 91.802, "p95_ms": 83.479}}, "truncated_count": 0}, "processing_total": {"avg_ms": 140.663, "count": 171, "max_ms": 193.078, "p95_ms": 171.073}, "read_interval": {"avg_ms": 152.031, "count": 170, "max_ms": 203.436, "p95_ms": 182.517}, "triton_wait": {"avg_ms": 63.979, "count": 171, "max_ms": 105.474, "p95_ms": 88.138}, "visualize": {"avg_ms": 6.229, "count": 171, "max_ms": 8.856, "p95_ms": 7.707}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.894, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1968}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5411.738, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5589372fe380] error while decoding MB 29 20, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 46, "h264_decode_error_count": 95, "h264_decode_error_count_per_hour": 23, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T17:06:01+09:00", "last_error_summary": "[h264 @ 0x5589372fe380] error while decoding MB 29 20, bytestream -5", "last_exit_at": "2026-07-02T17:03:21+09:00", "last_returncode": 0, "last_stable_duration_sec": 159.857, "last_started_at": "2026-07-02T17:03:21+09:00", "pid": 14286, "process": {"exists": false, "pid": 14286}, "restart_count": 59, "restart_count_per_hour": 14, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 60}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 179.619, "sample_count": 2009, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 236212, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T17:06:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-02T17:06:01+09:00", "last_frame_read_age_sec": 0.049, "last_frame_read_at": "2026-07-02T17:06:01+09:00", "mjpeg": {"age_sec": 0.047, "exists": true, "mtime": "2026-07-02T17:06:01.132155+09:00", "mtime_epoch": 1782979561.1321554, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185146}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T17:06:01+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1396.148, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 232, "pushed_frames": 232, "read_frames": 232, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.737, "pushed": 7.737, "read": 7.737}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.459, "count": 231, "max_ms": 9.909, "p95_ms": 8.991}, "gstreamer_push": {"avg_ms": 1.126, "count": 232, "max_ms": 1.906, "p95_ms": 1.402}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.291, "count": 231, "max_ms": 74.468, "p95_ms": 66.613}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.849, "count": 231, "max_ms": 111.337, "p95_ms": 89.358}, "yolo11_640_fire_11class": {"avg_ms": 50.561, "count": 231, "max_ms": 72.143, "p95_ms": 66.072}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.604, "count": 231, "max_ms": 95.188, "p95_ms": 86.014}}, "truncated_count": 0}, "processing_total": {"avg_ms": 117.198, "count": 232, "max_ms": 188.249, "p95_ms": 161.007}, "read_interval": {"avg_ms": 129.786, "count": 232, "max_ms": 197.967, "p95_ms": 171.423}, "triton_wait": {"avg_ms": 65.559, "count": 231, "max_ms": 111.349, "p95_ms": 89.602}, "visualize": {"avg_ms": 5.137, "count": 231, "max_ms": 8.913, "p95_ms": 7.502}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.985, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1968}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5410.828, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T17:06:01+09:00", "last_error_summary": "[h264 @ 0x5589372fe380] error while decoding MB 29 20, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 46, "h264_decode_error_count": 95, "h264_decode_error_count_per_hour": 23, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T17:06:01+09:00", "last_error_summary": "[h264 @ 0x5589372fe380] error while decoding MB 29 20, bytestream -5", "last_exit_at": "2026-07-02T17:06:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 159.924, "last_started_at": "2026-07-02T17:03:21+09:00", "pid": 14286, "process": {"exists": false, "pid": 14286}, "restart_count": 59, "restart_count_per_hour": 14, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 60}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 179.619, "sample_count": 2009, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 236213, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T17:06:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T17:06:01+09:00", "last_frame_read_age_sec": 0.117, "last_frame_read_at": "2026-07-02T17:06:01+09:00", "mjpeg": {"age_sec": 0.114, "exists": true, "mtime": "2026-07-02T17:06:01.132155+09:00", "mtime_epoch": 1782979561.1321554, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185146}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T17:06:01+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1396.148, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 232, "pushed_frames": 232, "read_frames": 231, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.759, "pushed": 7.759, "read": 7.726}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.449, "count": 232, "max_ms": 9.909, "p95_ms": 8.991}, "gstreamer_push": {"avg_ms": 1.127, "count": 232, "max_ms": 1.906, "p95_ms": 1.402}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.332, "count": 232, "max_ms": 74.468, "p95_ms": 66.613}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.984, "count": 232, "max_ms": 111.337, "p95_ms": 89.591}, "yolo11_640_fire_11class": {"avg_ms": 50.606, "count": 232, "max_ms": 72.143, "p95_ms": 66.072}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.647, "count": 232, "max_ms": 95.188, "p95_ms": 86.014}}, "truncated_count": 0}, "processing_total": {"avg_ms": 116.907, "count": 232, "max_ms": 188.249, "p95_ms": 160.736}, "read_interval": {"avg_ms": 129.543, "count": 231, "max_ms": 197.967, "p95_ms": 171.344}, "triton_wait": {"avg_ms": 65.683, "count": 232, "max_ms": 111.349, "p95_ms": 90.269}, "visualize": {"avg_ms": 5.131, "count": 232, "max_ms": 8.913, "p95_ms": 7.502}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.899, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1968}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5410.82, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 46, "h264_decode_error_count": 95, "h264_decode_error_count_per_hour": 23, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T17:06:01+09:00", "last_error_summary": "[h264 @ 0x5589372fe380] error while decoding MB 29 20, bytestream -5", "last_exit_at": "2026-07-02T17:06:01+09:00", "last_returncode": 0, "last_stable_duration_sec": 159.933, "last_started_at": "2026-07-02T17:03:21+09:00", "pid": 14286, "process": {"exists": false, "pid": 14286}, "restart_count": 59, "restart_count_per_hour": 14, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 60}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 179.619, "sample_count": 2009, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 236213, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T17:06:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-02T17:06:01+09:00", "last_frame_read_age_sec": 0.125, "last_frame_read_at": "2026-07-02T17:06:01+09:00", "mjpeg": {"age_sec": 0.122, "exists": true, "mtime": "2026-07-02T17:06:01.132155+09:00", "mtime_epoch": 1782979561.1321554, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185146}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T17:06:01+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1396.148, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 232, "pushed_frames": 232, "read_frames": 231, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.757, "pushed": 7.757, "read": 7.724}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.449, "count": 232, "max_ms": 9.909, "p95_ms": 8.991}, "gstreamer_push": {"avg_ms": 1.127, "count": 232, "max_ms": 1.906, "p95_ms": 1.402}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.332, "count": 232, "max_ms": 74.468, "p95_ms": 66.613}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.984, "count": 232, "max_ms": 111.337, "p95_ms": 89.591}, "yolo11_640_fire_11class": {"avg_ms": 50.606, "count": 232, "max_ms": 72.143, "p95_ms": 66.072}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.647, "count": 232, "max_ms": 95.188, "p95_ms": 86.014}}, "truncated_count": 0}, "processing_total": {"avg_ms": 116.907, "count": 232, "max_ms": 188.249, "p95_ms": 160.736}, "read_interval": {"avg_ms": 129.543, "count": 231, "max_ms": 197.967, "p95_ms": 171.344}, "triton_wait": {"avg_ms": 65.683, "count": 232, "max_ms": 111.349, "p95_ms": 90.269}, "visualize": {"avg_ms": 5.131, "count": 232, "max_ms": 8.913, "p95_ms": 7.502}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.908, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1968}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5530.895, "limit_mb": 8796093022207.996}, "detail": {"pid": 14393, "restart_count": 60, "start_count": 61}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 46, "h264_decode_error_count": 95, "h264_decode_error_count_per_hour": 23, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T17:06:01+09:00", "last_error_summary": "[h264 @ 0x5589372fe380] error while decoding MB 29 20, bytestream -5", "last_exit_at": "2026-07-02T17:06:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T17:06:03+09:00", "pid": 14393, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.004, "name": "ffmpeg", "pid": 14393, "rss_mb": 139.004, "state": "S (sleeping)", "threads": 106}, "restart_count": 60, "restart_count_per_hour": 15, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 61}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 179.619, "sample_count": 1977, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 236213, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T17:06:01+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.593, "last_frame_pushed_at": "2026-07-02T17:06:01+09:00", "last_frame_read_age_sec": 4.702, "last_frame_read_at": "2026-07-02T17:06:01+09:00", "mjpeg": {"age_sec": 4.579, "exists": true, "mtime": "2026-07-02T17:06:01.252158+09:00", "mtime_epoch": 1782979561.252158, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 185356}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T17:06:05+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1396.148, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 200, "pushed_frames": 200, "read_frames": 199, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.701, "pushed": 6.701, "read": 6.667}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.256, "count": 200, "max_ms": 9.686, "p95_ms": 8.71}, "gstreamer_push": {"avg_ms": 1.136, "count": 200, "max_ms": 1.906, "p95_ms": 1.407}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.344, "count": 200, "max_ms": 74.468, "p95_ms": 66.909}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.791, "count": 200, "max_ms": 111.337, "p95_ms": 90.257}, "yolo11_640_fire_11class": {"avg_ms": 50.711, "count": 200, "max_ms": 72.143, "p95_ms": 66.808}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.662, "count": 200, "max_ms": 95.188, "p95_ms": 86.282}}, "truncated_count": 0}, "processing_total": {"avg_ms": 114.364, "count": 200, "max_ms": 170.496, "p95_ms": 158.162}, "read_interval": {"avg_ms": 127.153, "count": 199, "max_ms": 183.025, "p95_ms": 170.769}, "triton_wait": {"avg_ms": 66.052, "count": 200, "max_ms": 111.349, "p95_ms": 91.377}, "visualize": {"avg_ms": 5.014, "count": 200, "max_ms": 8.913, "p95_ms": 6.972}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.847, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 2008}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5363.781, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x557c6c248000] error while decoding MB 68 25, bytestream -5 [in#0/rtsp @ 0x557c6c200e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557c6c200e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 34, "h264_decode_error_count": 35, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T17:08:17+09:00", "last_error_summary": "[h264 @ 0x557c6c248000] error while decoding MB 68 25, bytestream -5 [in#0/rtsp @ 0x557c6c200e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557c6c200e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T16:50:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 1040.492, "last_started_at": "2026-07-02T16:50:57+09:00", "pid": 13706, "process": {"exists": false, "pid": 13706}, "restart_count": 33, "restart_count_per_hour": 3, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 34}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 135.035, "sample_count": 2549, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 240973, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T17:08:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.076, "last_frame_pushed_at": "2026-07-02T17:08:17+09:00", "last_frame_read_age_sec": 0.066, "last_frame_read_at": "2026-07-02T17:08:17+09:00", "mjpeg": {"age_sec": 0.057, "exists": true, "mtime": "2026-07-02T17:08:17.686956+09:00", "mtime_epoch": 1782979697.686956, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 262718}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T17:08:17+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1459.445, "name": "celery", "pid": 267, "rss_mb": 1148.863, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 14, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.71, "pushed": 0.71, "read": 0.765}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 8.594, "count": 14, "max_ms": 9.383, "p95_ms": 9.383}, "gstreamer_push": {"avg_ms": 1.121, "count": 13, "max_ms": 1.316, "p95_ms": 1.316}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.418, "count": 14, "max_ms": 74.09, "p95_ms": 74.09}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.523, "count": 14, "max_ms": 88.221, "p95_ms": 88.221}, "yolo11_640_fire_11class": {"avg_ms": 52.991, "count": 14, "max_ms": 72.657, "p95_ms": 72.657}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.462, "count": 14, "max_ms": 99.289, "p95_ms": 99.289}}, "truncated_count": 0}, "processing_total": {"avg_ms": 110.3, "count": 13, "max_ms": 152.889, "p95_ms": 152.889}, "read_interval": {"avg_ms": 2165.452, "count": 14, "max_ms": 16727.801, "p95_ms": 16727.801}, "triton_wait": {"avg_ms": 64.21, "count": 14, "max_ms": 99.301, "p95_ms": 99.301}, "visualize": {"avg_ms": 4.638, "count": 14, "max_ms": 5.332, "p95_ms": 5.332}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.306, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 2008}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5363.301, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T17:08:17+09:00", "last_error_summary": "[h264 @ 0x557c6c248000] error while decoding MB 68 25, bytestream -5 [in#0/rtsp @ 0x557c6c200e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557c6c200e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 34, "h264_decode_error_count": 35, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T17:08:17+09:00", "last_error_summary": "[h264 @ 0x557c6c248000] error while decoding MB 68 25, bytestream -5 [in#0/rtsp @ 0x557c6c200e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557c6c200e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T17:08:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 1040.506, "last_started_at": "2026-07-02T16:50:57+09:00", "pid": 13706, "process": {"exists": false, "pid": 13706}, "restart_count": 33, "restart_count_per_hour": 3, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 34}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 135.035, "sample_count": 2549, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 240973, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T17:08:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T17:08:17+09:00", "last_frame_read_age_sec": 0.08, "last_frame_read_at": "2026-07-02T17:08:17+09:00", "mjpeg": {"age_sec": 0.071, "exists": true, "mtime": "2026-07-02T17:08:17.686956+09:00", "mtime_epoch": 1782979697.686956, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 262718}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T17:08:17+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1459.445, "name": "celery", "pid": 267, "rss_mb": 1148.863, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.764, "pushed": 0.764, "read": 0.764}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 8.594, "count": 14, "max_ms": 9.383, "p95_ms": 9.383}, "gstreamer_push": {"avg_ms": 1.143, "count": 14, "max_ms": 1.433, "p95_ms": 1.433}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.418, "count": 14, "max_ms": 74.09, "p95_ms": 74.09}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.523, "count": 14, "max_ms": 88.221, "p95_ms": 88.221}, "yolo11_640_fire_11class": {"avg_ms": 52.991, "count": 14, "max_ms": 72.657, "p95_ms": 72.657}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.462, "count": 14, "max_ms": 99.289, "p95_ms": 99.289}}, "truncated_count": 0}, "processing_total": {"avg_ms": 107.563, "count": 14, "max_ms": 152.889, "p95_ms": 152.889}, "read_interval": {"avg_ms": 2165.452, "count": 14, "max_ms": 16727.801, "p95_ms": 16727.801}, "triton_wait": {"avg_ms": 64.21, "count": 14, "max_ms": 99.301, "p95_ms": 99.301}, "visualize": {"avg_ms": 4.638, "count": 14, "max_ms": 5.332, "p95_ms": 5.332}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.32, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 2008}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5363.238, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 34, "h264_decode_error_count": 35, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T17:08:17+09:00", "last_error_summary": "[h264 @ 0x557c6c248000] error while decoding MB 68 25, bytestream -5 [in#0/rtsp @ 0x557c6c200e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557c6c200e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T17:08:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 1040.513, "last_started_at": "2026-07-02T16:50:57+09:00", "pid": 13706, "process": {"exists": false, "pid": 13706}, "restart_count": 33, "restart_count_per_hour": 3, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 34}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 135.035, "sample_count": 2549, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 240973, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T17:08:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-02T17:08:17+09:00", "last_frame_read_age_sec": 0.088, "last_frame_read_at": "2026-07-02T17:08:17+09:00", "mjpeg": {"age_sec": 0.078, "exists": true, "mtime": "2026-07-02T17:08:17.686956+09:00", "mtime_epoch": 1782979697.686956, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 262718}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T17:08:17+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1459.445, "name": "celery", "pid": 267, "rss_mb": 1148.863, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 27, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.764, "pushed": 0.764, "read": 0.764}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 8.594, "count": 14, "max_ms": 9.383, "p95_ms": 9.383}, "gstreamer_push": {"avg_ms": 1.143, "count": 14, "max_ms": 1.433, "p95_ms": 1.433}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.418, "count": 14, "max_ms": 74.09, "p95_ms": 74.09}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.523, "count": 14, "max_ms": 88.221, "p95_ms": 88.221}, "yolo11_640_fire_11class": {"avg_ms": 52.991, "count": 14, "max_ms": 72.657, "p95_ms": 72.657}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.462, "count": 14, "max_ms": 99.289, "p95_ms": 99.289}}, "truncated_count": 0}, "processing_total": {"avg_ms": 107.563, "count": 14, "max_ms": 152.889, "p95_ms": 152.889}, "read_interval": {"avg_ms": 2165.452, "count": 14, "max_ms": 16727.801, "p95_ms": 16727.801}, "triton_wait": {"avg_ms": 64.21, "count": 14, "max_ms": 99.301, "p95_ms": 99.301}, "visualize": {"avg_ms": 4.638, "count": 14, "max_ms": 5.332, "p95_ms": 5.332}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 18.328, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 2008}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5542.871, "limit_mb": 8796093022207.996}, "detail": {"pid": 14500, "restart_count": 34, "start_count": 35}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 34, "h264_decode_error_count": 35, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T17:08:17+09:00", "last_error_summary": "[h264 @ 0x557c6c248000] error while decoding MB 68 25, bytestream -5 [in#0/rtsp @ 0x557c6c200e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x557c6c200e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T17:08:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T17:08:19+09:00", "pid": 14500, "process": {"exists": true, "fd_count": 5, "hwm_mb": 210.32, "name": "ffmpeg", "pid": 14500, "rss_mb": 210.32, "state": "S (sleeping)", "threads": 106}, "restart_count": 34, "restart_count_per_hour": 4, "rtsp_timeout_count": 48, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 35}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 135.05, "sample_count": 2504, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 240973, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T17:08:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 4.085, "last_frame_pushed_at": "2026-07-02T17:08:17+09:00", "last_frame_read_age_sec": 4.159, "last_frame_read_at": "2026-07-02T17:08:17+09:00", "mjpeg": {"age_sec": 4.066, "exists": true, "mtime": "2026-07-02T17:08:17.770958+09:00", "mtime_epoch": 1782979697.7709577, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 255592}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T17:08:21+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1459.445, "name": "celery", "pid": 267, "rss_mb": 1148.863, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 14, "pushed_frames": 14, "read_frames": 14, "select_timeouts": 23, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.625, "pushed": 0.625, "read": 0.625}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 8.594, "count": 14, "max_ms": 9.383, "p95_ms": 9.383}, "gstreamer_push": {"avg_ms": 1.143, "count": 14, "max_ms": 1.433, "p95_ms": 1.433}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 54.418, "count": 14, "max_ms": 74.09, "p95_ms": 74.09}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 55.523, "count": 14, "max_ms": 88.221, "p95_ms": 88.221}, "yolo11_640_fire_11class": {"avg_ms": 52.991, "count": 14, "max_ms": 72.657, "p95_ms": 72.657}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.462, "count": 14, "max_ms": 99.289, "p95_ms": 99.289}}, "truncated_count": 0}, "processing_total": {"avg_ms": 107.563, "count": 14, "max_ms": 152.889, "p95_ms": 152.889}, "read_interval": {"avg_ms": 2165.452, "count": 14, "max_ms": 16727.801, "p95_ms": 16727.801}, "triton_wait": {"avg_ms": 64.21, "count": 14, "max_ms": 99.301, "p95_ms": 99.301}, "visualize": {"avg_ms": 4.638, "count": 14, "max_ms": 5.332, "p95_ms": 5.332}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 22.399, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1993}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5431.605, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x558c1c1f0500] error while decoding MB 67 7, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 47, "h264_decode_error_count": 96, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T17:11:54+09:00", "last_error_summary": "[h264 @ 0x558c1c1f0500] error while decoding MB 67 7, bytestream -5", "last_exit_at": "2026-07-02T17:06:03+09:00", "last_returncode": 0, "last_stable_duration_sec": 350.907, "last_started_at": "2026-07-02T17:06:03+09:00", "pid": 14393, "process": {"exists": false, "pid": 14393}, "restart_count": 60, "restart_count_per_hour": 14, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 61}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 151.158, "sample_count": 2542, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 239211, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T17:11:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.062, "last_frame_pushed_at": "2026-07-02T17:11:54+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-02T17:11:54+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-02T17:11:54.695207+09:00", "mtime_epoch": 1782979914.6952074, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196068}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T17:11:54+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1397.438, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 249, "pushed_frames": 249, "read_frames": 249, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.33, "pushed": 8.33, "read": 8.33}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.738, "count": 249, "max_ms": 10.064, "p95_ms": 7.644}, "gstreamer_push": {"avg_ms": 1.167, "count": 249, "max_ms": 1.824, "p95_ms": 1.399}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.126, "count": 249, "max_ms": 77.073, "p95_ms": 67.594}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.333, "count": 249, "max_ms": 102.517, "p95_ms": 82.933}, "yolo11_640_fire_11class": {"avg_ms": 50.827, "count": 249, "max_ms": 77.061, "p95_ms": 67.597}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.87, "count": 249, "max_ms": 98.856, "p95_ms": 87.241}}, "truncated_count": 0}, "processing_total": {"avg_ms": 106.361, "count": 249, "max_ms": 165.582, "p95_ms": 148.953}, "read_interval": {"avg_ms": 120.312, "count": 249, "max_ms": 176.771, "p95_ms": 162.571}, "triton_wait": {"avg_ms": 64.901, "count": 249, "max_ms": 102.529, "p95_ms": 87.257}, "visualize": {"avg_ms": 4.771, "count": 249, "max_ms": 7.176, "p95_ms": 6.283}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.892, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1993}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5430.965, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T17:11:54+09:00", "last_error_summary": "[h264 @ 0x558c1c1f0500] error while decoding MB 67 7, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 47, "h264_decode_error_count": 96, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T17:11:54+09:00", "last_error_summary": "[h264 @ 0x558c1c1f0500] error while decoding MB 67 7, bytestream -5", "last_exit_at": "2026-07-02T17:11:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 350.974, "last_started_at": "2026-07-02T17:06:03+09:00", "pid": 14393, "process": {"exists": false, "pid": 14393}, "restart_count": 60, "restart_count_per_hour": 14, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 61}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 151.158, "sample_count": 2543, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 239212, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T17:11:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T17:11:54+09:00", "last_frame_read_age_sec": 0.119, "last_frame_read_at": "2026-07-02T17:11:54+09:00", "mjpeg": {"age_sec": 0.11, "exists": true, "mtime": "2026-07-02T17:11:54.695207+09:00", "mtime_epoch": 1782979914.6952074, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196068}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T17:11:54+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1397.438, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 250, "pushed_frames": 250, "read_frames": 249, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.344, "pushed": 8.344, "read": 8.311}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.747, "count": 250, "max_ms": 10.064, "p95_ms": 7.666}, "gstreamer_push": {"avg_ms": 1.166, "count": 250, "max_ms": 1.824, "p95_ms": 1.399}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.115, "count": 250, "max_ms": 77.073, "p95_ms": 67.594}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.316, "count": 250, "max_ms": 102.517, "p95_ms": 82.933}, "yolo11_640_fire_11class": {"avg_ms": 50.794, "count": 250, "max_ms": 77.061, "p95_ms": 67.597}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.851, "count": 250, "max_ms": 98.856, "p95_ms": 87.241}}, "truncated_count": 0}, "processing_total": {"avg_ms": 106.384, "count": 250, "max_ms": 165.582, "p95_ms": 148.953}, "read_interval": {"avg_ms": 120.312, "count": 249, "max_ms": 176.771, "p95_ms": 162.571}, "triton_wait": {"avg_ms": 64.87, "count": 250, "max_ms": 102.529, "p95_ms": 87.257}, "visualize": {"avg_ms": 4.777, "count": 250, "max_ms": 7.176, "p95_ms": 6.286}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.96, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1993}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5430.957, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 47, "h264_decode_error_count": 96, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T17:11:54+09:00", "last_error_summary": "[h264 @ 0x558c1c1f0500] error while decoding MB 67 7, bytestream -5", "last_exit_at": "2026-07-02T17:11:54+09:00", "last_returncode": 0, "last_stable_duration_sec": 350.981, "last_started_at": "2026-07-02T17:06:03+09:00", "pid": 14393, "process": {"exists": false, "pid": 14393}, "restart_count": 60, "restart_count_per_hour": 14, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 61}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 151.158, "sample_count": 2543, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 239212, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T17:11:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-02T17:11:54+09:00", "last_frame_read_age_sec": 0.125, "last_frame_read_at": "2026-07-02T17:11:54+09:00", "mjpeg": {"age_sec": 0.116, "exists": true, "mtime": "2026-07-02T17:11:54.695207+09:00", "mtime_epoch": 1782979914.6952074, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196068}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T17:11:54+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1397.438, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 250, "pushed_frames": 250, "read_frames": 249, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.343, "pushed": 8.343, "read": 8.309}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.746, "count": 249, "max_ms": 10.064, "p95_ms": 7.666}, "gstreamer_push": {"avg_ms": 1.166, "count": 250, "max_ms": 1.824, "p95_ms": 1.399}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.108, "count": 249, "max_ms": 77.073, "p95_ms": 67.594}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.303, "count": 249, "max_ms": 102.517, "p95_ms": 82.933}, "yolo11_640_fire_11class": {"avg_ms": 50.792, "count": 249, "max_ms": 77.061, "p95_ms": 67.597}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 61.832, "count": 249, "max_ms": 98.856, "p95_ms": 87.241}}, "truncated_count": 0}, "processing_total": {"avg_ms": 106.384, "count": 250, "max_ms": 165.582, "p95_ms": 148.953}, "read_interval": {"avg_ms": 120.312, "count": 249, "max_ms": 176.771, "p95_ms": 162.571}, "triton_wait": {"avg_ms": 64.862, "count": 249, "max_ms": 102.529, "p95_ms": 87.257}, "visualize": {"avg_ms": 4.776, "count": 249, "max_ms": 7.176, "p95_ms": 6.286}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.966, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 1993}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5537.344, "limit_mb": 8796093022207.996}, "detail": {"pid": 14652, "restart_count": 61, "start_count": 62}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 47, "h264_decode_error_count": 96, "h264_decode_error_count_per_hour": 24, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T17:11:54+09:00", "last_error_summary": "[h264 @ 0x558c1c1f0500] error while decoding MB 67 7, bytestream -5", "last_exit_at": "2026-07-02T17:11:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T17:11:57+09:00", "pid": 14652, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.918, "name": "ffmpeg", "pid": 14652, "rss_mb": 141.918, "state": "S (sleeping)", "threads": 106}, "restart_count": 61, "restart_count_per_hour": 15, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 62}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 151.361, "sample_count": 2500, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 239212, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T17:11:54+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.275, "last_frame_pushed_at": "2026-07-02T17:11:54+09:00", "last_frame_read_age_sec": 4.388, "last_frame_read_at": "2026-07-02T17:11:54+09:00", "mjpeg": {"age_sec": 4.255, "exists": true, "mtime": "2026-07-02T17:11:54.819210+09:00", "mtime_epoch": 1782979914.8192096, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 196247}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T17:11:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1397.438, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 220, "pushed_frames": 220, "read_frames": 219, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.362, "pushed": 7.362, "read": 7.329}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.706, "count": 220, "max_ms": 10.064, "p95_ms": 7.667}, "gstreamer_push": {"avg_ms": 1.177, "count": 220, "max_ms": 1.824, "p95_ms": 1.411}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.965, "count": 220, "max_ms": 77.073, "p95_ms": 66.435}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.453, "count": 220, "max_ms": 102.517, "p95_ms": 81.693}, "yolo11_640_fire_11class": {"avg_ms": 49.593, "count": 220, "max_ms": 77.061, "p95_ms": 65.79}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.557, "count": 220, "max_ms": 97.379, "p95_ms": 86.108}}, "truncated_count": 0}, "processing_total": {"avg_ms": 102.987, "count": 220, "max_ms": 165.582, "p95_ms": 146.429}, "read_interval": {"avg_ms": 117.006, "count": 219, "max_ms": 176.771, "p95_ms": 162.571}, "triton_wait": {"avg_ms": 62.841, "count": 220, "max_ms": 102.529, "p95_ms": 86.464}, "visualize": {"avg_ms": 4.72, "count": 220, "max_ms": 7.176, "p95_ms": 6.286}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.883, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 2021}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5356.965, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x56410251c740] error while decoding MB 66 56, bytestream -11 [in#0/rtsp @ 0x56410237ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56410237ce00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 35, "h264_decode_error_count": 36, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T17:12:12+09:00", "last_error_summary": "[h264 @ 0x56410251c740] error while decoding MB 66 56, bytestream -11 [in#0/rtsp @ 0x56410237ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56410237ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T17:08:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 232.176, "last_started_at": "2026-07-02T17:08:19+09:00", "pid": 14500, "process": {"exists": false, "pid": 14500}, "restart_count": 34, "restart_count_per_hour": 4, "rtsp_timeout_count": 50, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 35}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.237, "sample_count": 1911, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 242581, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T17:12:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.075, "last_frame_pushed_at": "2026-07-02T17:12:11+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-02T17:12:11+09:00", "mjpeg": {"age_sec": 0.052, "exists": true, "mtime": "2026-07-02T17:12:11.959537+09:00", "mtime_epoch": 1782979931.9595373, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 256128}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T17:12:12+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1459.445, "name": "celery", "pid": 267, "rss_mb": 1149.637, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 26, "pushed_frames": 26, "read_frames": 26, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.868, "pushed": 0.868, "read": 0.868}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 9.476, "count": 25, "max_ms": 14.861, "p95_ms": 12.897}, "gstreamer_push": {"avg_ms": 1.037, "count": 26, "max_ms": 1.29, "p95_ms": 1.238}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.741, "count": 25, "max_ms": 74.379, "p95_ms": 73.416}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 51.53, "count": 25, "max_ms": 84.068, "p95_ms": 81.856}, "yolo11_640_fire_11class": {"avg_ms": 49.876, "count": 25, "max_ms": 64.448, "p95_ms": 63.014}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.862, "count": 25, "max_ms": 82.032, "p95_ms": 80.881}}, "truncated_count": 0}, "processing_total": {"avg_ms": 101.887, "count": 26, "max_ms": 137.735, "p95_ms": 127.484}, "read_interval": {"avg_ms": 1154.273, "count": 26, "max_ms": 17143.401, "p95_ms": 10165.078}, "triton_wait": {"avg_ms": 58.76, "count": 25, "max_ms": 84.08, "p95_ms": 81.869}, "visualize": {"avg_ms": 4.516, "count": 25, "max_ms": 6.012, "p95_ms": 5.571}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.963, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 2021}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5356.648, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T17:12:12+09:00", "last_error_summary": "[h264 @ 0x56410251c740] error while decoding MB 66 56, bytestream -11 [in#0/rtsp @ 0x56410237ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56410237ce00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 35, "h264_decode_error_count": 36, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T17:12:12+09:00", "last_error_summary": "[h264 @ 0x56410251c740] error while decoding MB 66 56, bytestream -11 [in#0/rtsp @ 0x56410237ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56410237ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T17:12:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 232.211, "last_started_at": "2026-07-02T17:08:19+09:00", "pid": 14500, "process": {"exists": false, "pid": 14500}, "restart_count": 34, "restart_count_per_hour": 4, "rtsp_timeout_count": 50, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 35}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.237, "sample_count": 1911, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 242582, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T17:12:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-02T17:12:12+09:00", "last_frame_read_age_sec": 0.099, "last_frame_read_at": "2026-07-02T17:12:11+09:00", "mjpeg": {"age_sec": 0.087, "exists": true, "mtime": "2026-07-02T17:12:11.959537+09:00", "mtime_epoch": 1782979931.9595373, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 256128}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T17:12:12+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1459.445, "name": "celery", "pid": 267, "rss_mb": 1149.637, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 27, "pushed_frames": 27, "read_frames": 26, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.9, "pushed": 0.9, "read": 0.867}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 9.385, "count": 26, "max_ms": 14.861, "p95_ms": 12.897}, "gstreamer_push": {"avg_ms": 1.044, "count": 27, "max_ms": 1.29, "p95_ms": 1.238}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.895, "count": 26, "max_ms": 74.379, "p95_ms": 73.416}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.307, "count": 26, "max_ms": 84.068, "p95_ms": 81.856}, "yolo11_640_fire_11class": {"avg_ms": 49.862, "count": 26, "max_ms": 64.448, "p95_ms": 63.014}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.819, "count": 26, "max_ms": 82.032, "p95_ms": 80.881}}, "truncated_count": 0}, "processing_total": {"avg_ms": 101.399, "count": 27, "max_ms": 137.735, "p95_ms": 127.484}, "read_interval": {"avg_ms": 1154.273, "count": 26, "max_ms": 17143.401, "p95_ms": 10165.078}, "triton_wait": {"avg_ms": 59.26, "count": 26, "max_ms": 84.08, "p95_ms": 81.869}, "visualize": {"avg_ms": 4.482, "count": 26, "max_ms": 6.012, "p95_ms": 5.571}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 2021}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5356.574, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 35, "h264_decode_error_count": 36, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T17:12:12+09:00", "last_error_summary": "[h264 @ 0x56410251c740] error while decoding MB 66 56, bytestream -11 [in#0/rtsp @ 0x56410237ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56410237ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T17:12:12+09:00", "last_returncode": 0, "last_stable_duration_sec": 232.218, "last_started_at": "2026-07-02T17:08:19+09:00", "pid": 14500, "process": {"exists": false, "pid": 14500}, "restart_count": 34, "restart_count_per_hour": 4, "rtsp_timeout_count": 50, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 35}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.237, "sample_count": 1911, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 242582, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T17:12:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "last_stable_duration_sec<600"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.014, "last_frame_pushed_at": "2026-07-02T17:12:12+09:00", "last_frame_read_age_sec": 0.105, "last_frame_read_at": "2026-07-02T17:12:11+09:00", "mjpeg": {"age_sec": 0.093, "exists": true, "mtime": "2026-07-02T17:12:11.959537+09:00", "mtime_epoch": 1782979931.9595373, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 256128}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T17:12:12+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1459.445, "name": "celery", "pid": 267, "rss_mb": 1149.637, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 26, "pushed_frames": 26, "read_frames": 26, "select_timeouts": 26, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.867, "pushed": 0.867, "read": 0.867}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 9.385, "count": 26, "max_ms": 14.861, "p95_ms": 12.897}, "gstreamer_push": {"avg_ms": 1.045, "count": 26, "max_ms": 1.29, "p95_ms": 1.238}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.895, "count": 26, "max_ms": 74.379, "p95_ms": 73.416}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 52.307, "count": 26, "max_ms": 84.068, "p95_ms": 81.856}, "yolo11_640_fire_11class": {"avg_ms": 49.862, "count": 26, "max_ms": 64.448, "p95_ms": 63.014}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 56.819, "count": 26, "max_ms": 82.032, "p95_ms": 80.881}}, "truncated_count": 0}, "processing_total": {"avg_ms": 100.999, "count": 26, "max_ms": 137.735, "p95_ms": 127.484}, "read_interval": {"avg_ms": 1154.273, "count": 26, "max_ms": 17143.401, "p95_ms": 10165.078}, "triton_wait": {"avg_ms": 59.26, "count": 26, "max_ms": 84.08, "p95_ms": 81.869}, "visualize": {"avg_ms": 4.482, "count": 26, "max_ms": 6.012, "p95_ms": 5.571}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.99, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 2021}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5478.555, "limit_mb": 8796093022207.996}, "detail": {"pid": 14759, "restart_count": 35, "start_count": 36}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 35, "h264_decode_error_count": 36, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T17:12:12+09:00", "last_error_summary": "[h264 @ 0x56410251c740] error while decoding MB 66 56, bytestream -11 [in#0/rtsp @ 0x56410237ce00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x56410237ce00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T17:12:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T17:12:14+09:00", "pid": 14759, "process": {"exists": true, "fd_count": 5, "hwm_mb": 156.387, "name": "ffmpeg", "pid": 14759, "rss_mb": 156.223, "state": "R (running)", "threads": 49}, "restart_count": 35, "restart_count_per_hour": 5, "rtsp_timeout_count": 50, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 36}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.237, "sample_count": 1876, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 242582, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T17:12:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 4.466, "last_frame_pushed_at": "2026-07-02T17:12:12+09:00", "last_frame_read_age_sec": 4.557, "last_frame_read_at": "2026-07-02T17:12:11+09:00", "mjpeg": {"age_sec": 4.445, "exists": true, "mtime": "2026-07-02T17:12:12.059539+09:00", "mtime_epoch": 1782979932.059539, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 255811}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T17:12:16+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1459.445, "name": "celery", "pid": 267, "rss_mb": 1149.637, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 13, "pushed_frames": 13, "read_frames": 13, "select_timeouts": 24, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 0.823, "pushed": 0.823, "read": 0.823}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 8.163, "count": 13, "max_ms": 12.425, "p95_ms": 12.425}, "gstreamer_push": {"avg_ms": 1.106, "count": 13, "max_ms": 1.29, "p95_ms": 1.29}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 55.186, "count": 13, "max_ms": 74.379, "p95_ms": 74.379}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.75, "count": 13, "max_ms": 84.068, "p95_ms": 84.068}, "yolo11_640_fire_11class": {"avg_ms": 51.355, "count": 13, "max_ms": 64.448, "p95_ms": 64.448}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.581, "count": 13, "max_ms": 82.032, "p95_ms": 82.032}}, "truncated_count": 0}, "processing_total": {"avg_ms": 87.597, "count": 13, "max_ms": 107.38, "p95_ms": 107.38}, "read_interval": {"avg_ms": 2183.965, "count": 13, "max_ms": 17143.401, "p95_ms": 17143.401}, "triton_wait": {"avg_ms": 62.038, "count": 13, "max_ms": 84.08, "p95_ms": 84.08}, "visualize": {"avg_ms": 3.828, "count": 13, "max_ms": 5.571, "p95_ms": 5.571}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 15.805, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2015}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5434.305, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5568c8090c80] error while decoding MB 19 37, bytestream -7", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 48, "h264_decode_error_count": 97, "h264_decode_error_count_per_hour": 22, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T17:17:15+09:00", "last_error_summary": "[h264 @ 0x5568c8090c80] error while decoding MB 19 37, bytestream -7", "last_exit_at": "2026-07-02T17:11:57+09:00", "last_returncode": 0, "last_stable_duration_sec": 318.865, "last_started_at": "2026-07-02T17:11:57+09:00", "pid": 14652, "process": {"exists": false, "pid": 14652}, "restart_count": 61, "restart_count_per_hour": 14, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 62}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.35, "sample_count": 2527, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 241894, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T17:17:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.066, "last_frame_pushed_at": "2026-07-02T17:17:15+09:00", "last_frame_read_age_sec": 0.054, "last_frame_read_at": "2026-07-02T17:17:15+09:00", "mjpeg": {"age_sec": 0.05, "exists": true, "mtime": "2026-07-02T17:17:15.889206+09:00", "mtime_epoch": 1782980235.8892064, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191569}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T17:17:15+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1398.727, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 281, "pushed_frames": 281, "read_frames": 281, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 9.389, "pushed": 9.389, "read": 9.389}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.102, "count": 281, "max_ms": 9.475, "p95_ms": 7.077}, "gstreamer_push": {"avg_ms": 1.222, "count": 281, "max_ms": 1.485, "p95_ms": 1.408}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.009, "count": 281, "max_ms": 75.787, "p95_ms": 66.706}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.387, "count": 281, "max_ms": 105.649, "p95_ms": 84.44}, "yolo11_640_fire_11class": {"avg_ms": 50.197, "count": 281, "max_ms": 76.38, "p95_ms": 66.128}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.002, "count": 281, "max_ms": 95.176, "p95_ms": 84.924}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.954, "count": 281, "max_ms": 161.447, "p95_ms": 135.255}, "read_interval": {"avg_ms": 106.627, "count": 281, "max_ms": 177.474, "p95_ms": 150.28}, "triton_wait": {"avg_ms": 63.467, "count": 281, "max_ms": 105.662, "p95_ms": 88.102}, "visualize": {"avg_ms": 4.291, "count": 281, "max_ms": 7.056, "p95_ms": 5.854}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.928, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2015}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5434.055, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T17:17:15+09:00", "last_error_summary": "[h264 @ 0x5568c8090c80] error while decoding MB 19 37, bytestream -7", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 48, "h264_decode_error_count": 97, "h264_decode_error_count_per_hour": 22, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T17:17:15+09:00", "last_error_summary": "[h264 @ 0x5568c8090c80] error while decoding MB 19 37, bytestream -7", "last_exit_at": "2026-07-02T17:17:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 318.898, "last_started_at": "2026-07-02T17:11:57+09:00", "pid": 14652, "process": {"exists": false, "pid": 14652}, "restart_count": 61, "restart_count_per_hour": 14, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 62}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.35, "sample_count": 2527, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 241895, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T17:17:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-02T17:17:15+09:00", "last_frame_read_age_sec": 0.086, "last_frame_read_at": "2026-07-02T17:17:15+09:00", "mjpeg": {"age_sec": 0.082, "exists": true, "mtime": "2026-07-02T17:17:15.889206+09:00", "mtime_epoch": 1782980235.8892064, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191569}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T17:17:15+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1398.727, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 282, "pushed_frames": 282, "read_frames": 281, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 9.413, "pushed": 9.413, "read": 9.379}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.099, "count": 282, "max_ms": 9.475, "p95_ms": 7.077}, "gstreamer_push": {"avg_ms": 1.222, "count": 282, "max_ms": 1.485, "p95_ms": 1.408}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.002, "count": 282, "max_ms": 75.787, "p95_ms": 66.706}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.387, "count": 282, "max_ms": 105.649, "p95_ms": 84.44}, "yolo11_640_fire_11class": {"avg_ms": 50.202, "count": 282, "max_ms": 76.38, "p95_ms": 66.128}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.0, "count": 282, "max_ms": 95.176, "p95_ms": 84.924}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.898, "count": 282, "max_ms": 161.447, "p95_ms": 135.255}, "read_interval": {"avg_ms": 106.627, "count": 281, "max_ms": 177.474, "p95_ms": 150.28}, "triton_wait": {"avg_ms": 63.455, "count": 282, "max_ms": 105.662, "p95_ms": 88.102}, "visualize": {"avg_ms": 4.299, "count": 282, "max_ms": 7.056, "p95_ms": 5.918}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.96, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2015}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5433.785, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 48, "h264_decode_error_count": 97, "h264_decode_error_count_per_hour": 22, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T17:17:15+09:00", "last_error_summary": "[h264 @ 0x5568c8090c80] error while decoding MB 19 37, bytestream -7", "last_exit_at": "2026-07-02T17:17:15+09:00", "last_returncode": 0, "last_stable_duration_sec": 318.907, "last_started_at": "2026-07-02T17:11:57+09:00", "pid": 14652, "process": {"exists": false, "pid": 14652}, "restart_count": 61, "restart_count_per_hour": 14, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 62}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.35, "sample_count": 2527, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 241895, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T17:17:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.017, "last_frame_pushed_at": "2026-07-02T17:17:15+09:00", "last_frame_read_age_sec": 0.096, "last_frame_read_at": "2026-07-02T17:17:15+09:00", "mjpeg": {"age_sec": 0.092, "exists": true, "mtime": "2026-07-02T17:17:15.889206+09:00", "mtime_epoch": 1782980235.8892064, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 191569}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T17:17:15+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1398.727, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 282, "pushed_frames": 282, "read_frames": 281, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 9.41, "pushed": 9.41, "read": 9.376}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.099, "count": 282, "max_ms": 9.475, "p95_ms": 7.077}, "gstreamer_push": {"avg_ms": 1.222, "count": 282, "max_ms": 1.485, "p95_ms": 1.408}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.002, "count": 282, "max_ms": 75.787, "p95_ms": 66.706}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.387, "count": 282, "max_ms": 105.649, "p95_ms": 84.44}, "yolo11_640_fire_11class": {"avg_ms": 50.202, "count": 282, "max_ms": 76.38, "p95_ms": 66.128}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.0, "count": 282, "max_ms": 95.176, "p95_ms": 84.924}}, "truncated_count": 0}, "processing_total": {"avg_ms": 91.898, "count": 282, "max_ms": 161.447, "p95_ms": 135.255}, "read_interval": {"avg_ms": 106.627, "count": 281, "max_ms": 177.474, "p95_ms": 150.28}, "triton_wait": {"avg_ms": 63.455, "count": 282, "max_ms": 105.662, "p95_ms": 88.102}, "visualize": {"avg_ms": 4.299, "count": 282, "max_ms": 7.056, "p95_ms": 5.918}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.969, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2015}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5543.5, "limit_mb": 8796093022207.996}, "detail": {"pid": 14911, "restart_count": 62, "start_count": 63}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 48, "h264_decode_error_count": 97, "h264_decode_error_count_per_hour": 22, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T17:17:15+09:00", "last_error_summary": "[h264 @ 0x5568c8090c80] error while decoding MB 19 37, bytestream -7", "last_exit_at": "2026-07-02T17:17:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T17:17:18+09:00", "pid": 14911, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.492, "name": "ffmpeg", "pid": 14911, "rss_mb": 138.492, "state": "S (sleeping)", "threads": 106}, "restart_count": 62, "restart_count_per_hour": 15, "rtsp_timeout_count": 28, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 63}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 144.428, "sample_count": 2491, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 241895, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T17:17:15+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.168, "last_frame_pushed_at": "2026-07-02T17:17:15+09:00", "last_frame_read_age_sec": 4.246, "last_frame_read_at": "2026-07-02T17:17:15+09:00", "mjpeg": {"age_sec": 4.151, "exists": true, "mtime": "2026-07-02T17:17:15.981208+09:00", "mtime_epoch": 1782980235.981208, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 190865}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T17:17:20+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1398.727, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 240, "pushed_frames": 240, "read_frames": 239, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.014, "pushed": 8.014, "read": 7.98}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.099, "count": 240, "max_ms": 9.475, "p95_ms": 7.037}, "gstreamer_push": {"avg_ms": 1.219, "count": 240, "max_ms": 1.485, "p95_ms": 1.408}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.392, "count": 240, "max_ms": 75.787, "p95_ms": 66.826}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.669, "count": 240, "max_ms": 105.649, "p95_ms": 84.818}, "yolo11_640_fire_11class": {"avg_ms": 50.542, "count": 240, "max_ms": 76.38, "p95_ms": 66.696}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.647, "count": 240, "max_ms": 95.176, "p95_ms": 85.012}}, "truncated_count": 0}, "processing_total": {"avg_ms": 93.111, "count": 240, "max_ms": 161.447, "p95_ms": 137.512}, "read_interval": {"avg_ms": 107.826, "count": 239, "max_ms": 177.474, "p95_ms": 151.023}, "triton_wait": {"avg_ms": 64.142, "count": 240, "max_ms": 105.662, "p95_ms": 88.961}, "visualize": {"avg_ms": 4.307, "count": 240, "max_ms": 7.056, "p95_ms": 5.956}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.949, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 2138}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5398.703, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55715e311dc0] error while decoding MB 42 33, bytestream -21 [in#0/rtsp @ 0x55715e294e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55715e294e00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 36, "h264_decode_error_count": 37, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T17:38:17+09:00", "last_error_summary": "[h264 @ 0x55715e311dc0] error while decoding MB 42 33, bytestream -21 [in#0/rtsp @ 0x55715e294e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55715e294e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T17:12:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 1562.605, "last_started_at": "2026-07-02T17:12:14+09:00", "pid": 14759, "process": {"exists": false, "pid": 14759}, "restart_count": 35, "restart_count_per_hour": 4, "rtsp_timeout_count": 52, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 36}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.674, "sample_count": 2853, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 256590, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T17:38:16+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.074, "last_frame_pushed_at": "2026-07-02T17:38:17+09:00", "last_frame_read_age_sec": 0.06, "last_frame_read_at": "2026-07-02T17:38:17+09:00", "mjpeg": {"age_sec": 0.049, "exists": true, "mtime": "2026-07-02T17:38:17.059516+09:00", "mtime_epoch": 1782981497.0595162, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 262654}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T17:38:17+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1459.445, "name": "celery", "pid": 267, "rss_mb": 1156.27, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 115, "pushed_frames": 115, "read_frames": 115, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.84, "pushed": 3.84, "read": 3.84}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 7.281, "count": 115, "max_ms": 12.026, "p95_ms": 9.032}, "gstreamer_push": {"avg_ms": 1.252, "count": 115, "max_ms": 1.544, "p95_ms": 1.405}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.686, "count": 115, "max_ms": 71.507, "p95_ms": 64.77}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.214, "count": 115, "max_ms": 104.039, "p95_ms": 80.952}, "yolo11_640_fire_11class": {"avg_ms": 50.122, "count": 115, "max_ms": 75.872, "p95_ms": 63.354}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.828, "count": 115, "max_ms": 90.135, "p95_ms": 80.777}}, "truncated_count": 0}, "processing_total": {"avg_ms": 72.191, "count": 115, "max_ms": 121.295, "p95_ms": 98.324}, "read_interval": {"avg_ms": 260.467, "count": 115, "max_ms": 10164.483, "p95_ms": 117.259}, "triton_wait": {"avg_ms": 56.392, "count": 115, "max_ms": 104.051, "p95_ms": 82.049}, "visualize": {"avg_ms": 3.571, "count": 115, "max_ms": 4.705, "p95_ms": 4.159}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.951, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 2138}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5396.941, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T17:38:17+09:00", "last_error_summary": "[h264 @ 0x55715e311dc0] error while decoding MB 42 33, bytestream -21 [in#0/rtsp @ 0x55715e294e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55715e294e00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/01.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 36, "h264_decode_error_count": 37, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T17:38:17+09:00", "last_error_summary": "[h264 @ 0x55715e311dc0] error while decoding MB 42 33, bytestream -21 [in#0/rtsp @ 0x55715e294e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55715e294e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T17:38:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 1562.625, "last_started_at": "2026-07-02T17:12:14+09:00", "pid": 14759, "process": {"exists": false, "pid": 14759}, "restart_count": 35, "restart_count_per_hour": 4, "rtsp_timeout_count": 52, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 36}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.674, "sample_count": 2854, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 256591, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T17:38:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.007, "last_frame_pushed_at": "2026-07-02T17:38:17+09:00", "last_frame_read_age_sec": 0.08, "last_frame_read_at": "2026-07-02T17:38:17+09:00", "mjpeg": {"age_sec": 0.07, "exists": true, "mtime": "2026-07-02T17:38:17.059516+09:00", "mtime_epoch": 1782981497.0595162, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 262654}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T17:38:17+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1459.445, "name": "celery", "pid": 267, "rss_mb": 1156.27, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 115, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.87, "pushed": 3.87, "read": 3.837}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 7.281, "count": 116, "max_ms": 12.026, "p95_ms": 9.032}, "gstreamer_push": {"avg_ms": 1.252, "count": 116, "max_ms": 1.544, "p95_ms": 1.405}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.667, "count": 116, "max_ms": 71.507, "p95_ms": 64.77}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.18, "count": 116, "max_ms": 104.039, "p95_ms": 80.952}, "yolo11_640_fire_11class": {"avg_ms": 50.108, "count": 116, "max_ms": 75.872, "p95_ms": 63.354}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.782, "count": 116, "max_ms": 90.135, "p95_ms": 80.777}}, "truncated_count": 0}, "processing_total": {"avg_ms": 72.179, "count": 116, "max_ms": 121.295, "p95_ms": 98.324}, "read_interval": {"avg_ms": 260.467, "count": 115, "max_ms": 10164.483, "p95_ms": 117.259}, "triton_wait": {"avg_ms": 56.33, "count": 116, "max_ms": 104.051, "p95_ms": 82.049}, "visualize": {"avg_ms": 3.597, "count": 116, "max_ms": 6.593, "p95_ms": 4.182}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.972, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 2138}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5396.512, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 36, "h264_decode_error_count": 37, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T17:38:17+09:00", "last_error_summary": "[h264 @ 0x55715e311dc0] error while decoding MB 42 33, bytestream -21 [in#0/rtsp @ 0x55715e294e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55715e294e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T17:38:17+09:00", "last_returncode": 0, "last_stable_duration_sec": 1562.634, "last_started_at": "2026-07-02T17:12:14+09:00", "pid": 14759, "process": {"exists": false, "pid": 14759}, "restart_count": 35, "restart_count_per_hour": 4, "rtsp_timeout_count": 52, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 36}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.674, "sample_count": 2854, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 256591, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T17:38:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 0.016, "last_frame_pushed_at": "2026-07-02T17:38:17+09:00", "last_frame_read_age_sec": 0.089, "last_frame_read_at": "2026-07-02T17:38:17+09:00", "mjpeg": {"age_sec": 0.078, "exists": true, "mtime": "2026-07-02T17:38:17.059516+09:00", "mtime_epoch": 1782981497.0595162, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 262654}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T17:38:17+09:00", "process": {"exists": true, "fd_count": 134, "hwm_mb": 1459.445, "name": "celery", "pid": 267, "rss_mb": 1156.27, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 116, "pushed_frames": 116, "read_frames": 115, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.869, "pushed": 3.869, "read": 3.836}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 7.281, "count": 116, "max_ms": 12.026, "p95_ms": 9.032}, "gstreamer_push": {"avg_ms": 1.252, "count": 116, "max_ms": 1.544, "p95_ms": 1.405}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.667, "count": 116, "max_ms": 71.507, "p95_ms": 64.77}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 53.18, "count": 116, "max_ms": 104.039, "p95_ms": 80.952}, "yolo11_640_fire_11class": {"avg_ms": 50.108, "count": 116, "max_ms": 75.872, "p95_ms": 63.354}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 53.782, "count": 116, "max_ms": 90.135, "p95_ms": 80.777}}, "truncated_count": 0}, "processing_total": {"avg_ms": 72.179, "count": 116, "max_ms": 121.295, "p95_ms": 98.324}, "read_interval": {"avg_ms": 260.467, "count": 115, "max_ms": 10164.483, "p95_ms": 117.259}, "triton_wait": {"avg_ms": 56.33, "count": 116, "max_ms": 104.051, "p95_ms": 82.049}, "visualize": {"avg_ms": 3.597, "count": 116, "max_ms": 6.593, "p95_ms": 4.182}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.98, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/01/tracking_analysis_20260702_092040_909fc3a8.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 50, "tracking_summary": 2138}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "01", "cgroup_memory": {"current_mb": 5521.91, "limit_mb": 8796093022207.996}, "detail": {"pid": 15018, "restart_count": 36, "start_count": 37}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 36, "h264_decode_error_count": 37, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T17:38:17+09:00", "last_error_summary": "[h264 @ 0x55715e311dc0] error while decoding MB 42 33, bytestream -21 [in#0/rtsp @ 0x55715e294e00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55715e294e00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T17:38:19+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T17:38:19+09:00", "pid": 15018, "process": {"exists": true, "fd_count": 5, "hwm_mb": 157.16, "name": "ffmpeg", "pid": 15018, "rss_mb": 156.996, "state": "R (running)", "threads": 49}, "restart_count": 36, "restart_count_per_hour": 5, "rtsp_timeout_count": 52, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 37}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 104.781, "sample_count": 2813, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 256591, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T17:38:17+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "909fc3a8", "last_frame_pushed_age_sec": 4.4, "last_frame_pushed_at": "2026-07-02T17:38:17+09:00", "last_frame_read_age_sec": 4.473, "last_frame_read_at": "2026-07-02T17:38:17+09:00", "mjpeg": {"age_sec": 4.379, "exists": true, "mtime": "2026-07-02T17:38:17.143518+09:00", "mtime_epoch": 1782981497.1435177, "path": "/data/playlist/streaming/01/mjpeg.jpg", "size_bytes": 262508}, "monitor_url": "http://192.168.50.244:1223/mjpeg/01", "observed_at": "2026-07-02T17:38:21+09:00", "process": {"exists": true, "fd_count": 135, "hwm_mb": 1459.445, "name": "celery", "pid": 267, "rss_mb": 1156.016, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 2, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://admin:smart1357!@223.171.147.71:554/trackID=1", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 68, "pushed_frames": 68, "read_frames": 67, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.273, "pushed": 2.273, "read": 2.24}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 7.306, "count": 68, "max_ms": 12.026, "p95_ms": 8.757}, "gstreamer_push": {"avg_ms": 1.279, "count": 68, "max_ms": 1.544, "p95_ms": 1.405}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.257, "count": 68, "max_ms": 71.507, "p95_ms": 63.838}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 51.637, "count": 68, "max_ms": 104.039, "p95_ms": 82.412}, "yolo11_640_fire_11class": {"avg_ms": 48.555, "count": 68, "max_ms": 70.991, "p95_ms": 62.148}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 51.574, "count": 68, "max_ms": 90.135, "p95_ms": 76.337}}, "truncated_count": 0}, "processing_total": {"avg_ms": 70.431, "count": 68, "max_ms": 121.295, "p95_ms": 99.448}, "read_interval": {"avg_ms": 381.558, "count": 67, "max_ms": 10164.483, "p95_ms": 137.869}, "triton_wait": {"avg_ms": 54.316, "count": 68, "max_ms": 104.051, "p95_ms": 82.422}, "visualize": {"avg_ms": 3.686, "count": 68, "max_ms": 6.593, "p95_ms": 4.182}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.916, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "79f0ea2b-9015-4f2d-9eea-a195ac4c4e56", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2654}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5547.535, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 5, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55b60a8dcb00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b60a8dcb00] error while decoding MB 0 25, bytestream 3297 [h264 @ 0x55b60a9ff540] cabac decode of qscale diff failed at 43 11 [h264 @ 0x55b60a9ff540] error while decoding MB 43 11, bytestream 115911 [h264 @ 0x55b60b653940] error while decoding MB 55 22, bytestream -30 [h264 @ 0x55b60a911280] error while decoding MB 0 30, bytestream -5 [in#0/rtsp @ 0x55b60a895e00] Error during demuxing: Connection timed out ", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 49, "h264_decode_error_count": 102, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T19:25:34+09:00", "last_error_summary": "[h264 @ 0x55b60a8dcb00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b60a8dcb00] error while decoding MB 0 25, bytestream 3297 [h264 @ 0x55b60a9ff540] cabac decode of qscale diff failed at 43 11 [h264 @ 0x55b60a9ff540] error while decoding MB 43 11, bytestream 115911 [h264 @ 0x55b60b653940] error while decoding MB 55 22, bytestream -30 [h264 @ 0x55b60a911280] error while decoding MB 0 30, bytestream -5 [in#0/rtsp @ 0x55b60a895e00] Error during demuxing: Connection timed out ", "last_exit_at": "2026-07-02T17:17:18+09:00", "last_returncode": 0, "last_stable_duration_sec": 7696.624, "last_started_at": "2026-07-02T17:17:18+09:00", "pid": 14911, "process": {"exists": false, "pid": 14911}, "restart_count": 62, "restart_count_per_hour": 0, "rtsp_timeout_count": 30, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 63}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 130.386, "sample_count": 2714, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 318513, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T19:25:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.073, "last_frame_pushed_at": "2026-07-02T19:25:34+09:00", "last_frame_read_age_sec": 0.06, "last_frame_read_at": "2026-07-02T19:25:34+09:00", "mjpeg": {"age_sec": 0.057, "exists": true, "mtime": "2026-07-02T19:25:34.698601+09:00", "mtime_epoch": 1782987934.6986012, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 162208}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T19:25:34+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1436.215, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 113, "pushed_frames": 113, "read_frames": 113, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.773, "pushed": 3.773, "read": 3.773}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.243, "count": 113, "max_ms": 6.274, "p95_ms": 5.903}, "gstreamer_push": {"avg_ms": 1.295, "count": 113, "max_ms": 1.681, "p95_ms": 1.45}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.937, "count": 113, "max_ms": 73.445, "p95_ms": 67.017}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.609, "count": 113, "max_ms": 87.069, "p95_ms": 83.1}, "yolo11_640_fire_11class": {"avg_ms": 51.937, "count": 113, "max_ms": 73.227, "p95_ms": 67.353}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.447, "count": 113, "max_ms": 91.84, "p95_ms": 84.784}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.454, "count": 113, "max_ms": 133.698, "p95_ms": 102.093}, "read_interval": {"avg_ms": 265.348, "count": 113, "max_ms": 19206.733, "p95_ms": 120.309}, "triton_wait": {"avg_ms": 65.25, "count": 113, "max_ms": 91.851, "p95_ms": 84.796}, "visualize": {"avg_ms": 3.731, "count": 113, "max_ms": 5.25, "p95_ms": 4.534}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.947, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2654}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5547.078, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T19:25:34+09:00", "last_error_summary": "[h264 @ 0x55b60a8dcb00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b60a8dcb00] error while decoding MB 0 25, bytestream 3297 [h264 @ 0x55b60a9ff540] cabac decode of qscale diff failed at 43 11 [h264 @ 0x55b60a9ff540] error while decoding MB 43 11, bytestream 115911 [h264 @ 0x55b60b653940] error while decoding MB 55 22, bytestream -30 [h264 @ 0x55b60a911280] error while decoding MB 0 30, bytestream -5 [in#0/rtsp @ 0x55b60a895e00] Error during demuxing: Connection timed out ", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 49, "h264_decode_error_count": 102, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T19:25:34+09:00", "last_error_summary": "[h264 @ 0x55b60a8dcb00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b60a8dcb00] error while decoding MB 0 25, bytestream 3297 [h264 @ 0x55b60a9ff540] cabac decode of qscale diff failed at 43 11 [h264 @ 0x55b60a9ff540] error while decoding MB 43 11, bytestream 115911 [h264 @ 0x55b60b653940] error while decoding MB 55 22, bytestream -30 [h264 @ 0x55b60a911280] error while decoding MB 0 30, bytestream -5 [in#0/rtsp @ 0x55b60a895e00] Error during demuxing: Connection timed out ", "last_exit_at": "2026-07-02T19:25:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 7696.676, "last_started_at": "2026-07-02T17:17:18+09:00", "pid": 14911, "process": {"exists": false, "pid": 14911}, "restart_count": 62, "restart_count_per_hour": 0, "rtsp_timeout_count": 30, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 63}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 130.386, "sample_count": 2715, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 318514, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T19:25:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-02T19:25:34+09:00", "last_frame_read_age_sec": 0.112, "last_frame_read_at": "2026-07-02T19:25:34+09:00", "mjpeg": {"age_sec": 0.108, "exists": true, "mtime": "2026-07-02T19:25:34.698601+09:00", "mtime_epoch": 1782987934.6986012, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 162208}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T19:25:34+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1436.215, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 114, "pushed_frames": 114, "read_frames": 113, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.8, "pushed": 3.8, "read": 3.767}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.244, "count": 113, "max_ms": 6.274, "p95_ms": 5.903}, "gstreamer_push": {"avg_ms": 1.297, "count": 114, "max_ms": 1.681, "p95_ms": 1.463}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.789, "count": 113, "max_ms": 73.445, "p95_ms": 67.017}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.605, "count": 113, "max_ms": 87.069, "p95_ms": 83.1}, "yolo11_640_fire_11class": {"avg_ms": 51.754, "count": 113, "max_ms": 73.227, "p95_ms": 66.945}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.191, "count": 113, "max_ms": 91.84, "p95_ms": 84.367}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.627, "count": 114, "max_ms": 133.698, "p95_ms": 102.093}, "read_interval": {"avg_ms": 265.348, "count": 113, "max_ms": 19206.733, "p95_ms": 120.309}, "triton_wait": {"avg_ms": 65.247, "count": 113, "max_ms": 91.851, "p95_ms": 84.796}, "visualize": {"avg_ms": 3.733, "count": 113, "max_ms": 5.25, "p95_ms": 4.534}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.998, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2654}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5547.062, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 49, "h264_decode_error_count": 102, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T19:25:34+09:00", "last_error_summary": "[h264 @ 0x55b60a8dcb00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b60a8dcb00] error while decoding MB 0 25, bytestream 3297 [h264 @ 0x55b60a9ff540] cabac decode of qscale diff failed at 43 11 [h264 @ 0x55b60a9ff540] error while decoding MB 43 11, bytestream 115911 [h264 @ 0x55b60b653940] error while decoding MB 55 22, bytestream -30 [h264 @ 0x55b60a911280] error while decoding MB 0 30, bytestream -5 [in#0/rtsp @ 0x55b60a895e00] Error during demuxing: Connection timed out ", "last_exit_at": "2026-07-02T19:25:34+09:00", "last_returncode": 0, "last_stable_duration_sec": 7696.684, "last_started_at": "2026-07-02T17:17:18+09:00", "pid": 14911, "process": {"exists": false, "pid": 14911}, "restart_count": 62, "restart_count_per_hour": 0, "rtsp_timeout_count": 30, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 63}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 130.386, "sample_count": 2715, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 318514, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T19:25:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.019, "last_frame_pushed_at": "2026-07-02T19:25:34+09:00", "last_frame_read_age_sec": 0.12, "last_frame_read_at": "2026-07-02T19:25:34+09:00", "mjpeg": {"age_sec": 0.116, "exists": true, "mtime": "2026-07-02T19:25:34.698601+09:00", "mtime_epoch": 1782987934.6986012, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 162208}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T19:25:34+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1436.215, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 113, "pushed_frames": 113, "read_frames": 113, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.768, "pushed": 3.768, "read": 3.768}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.244, "count": 113, "max_ms": 6.274, "p95_ms": 5.903}, "gstreamer_push": {"avg_ms": 1.297, "count": 113, "max_ms": 1.681, "p95_ms": 1.463}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.789, "count": 113, "max_ms": 73.445, "p95_ms": 67.017}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.605, "count": 113, "max_ms": 87.069, "p95_ms": 83.1}, "yolo11_640_fire_11class": {"avg_ms": 51.754, "count": 113, "max_ms": 73.227, "p95_ms": 66.945}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.191, "count": 113, "max_ms": 91.84, "p95_ms": 84.367}}, "truncated_count": 0}, "processing_total": {"avg_ms": 79.461, "count": 113, "max_ms": 133.698, "p95_ms": 102.093}, "read_interval": {"avg_ms": 265.348, "count": 113, "max_ms": 19206.733, "p95_ms": 120.309}, "triton_wait": {"avg_ms": 65.247, "count": 113, "max_ms": 91.851, "p95_ms": 84.796}, "visualize": {"avg_ms": 3.733, "count": 113, "max_ms": 5.25, "p95_ms": 4.534}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.99, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2654}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5650.207, "limit_mb": 8796093022207.996}, "detail": {"pid": 15170, "restart_count": 63, "start_count": 64}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 49, "h264_decode_error_count": 102, "h264_decode_error_count_per_hour": 5, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T19:25:34+09:00", "last_error_summary": "[h264 @ 0x55b60a8dcb00] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55b60a8dcb00] error while decoding MB 0 25, bytestream 3297 [h264 @ 0x55b60a9ff540] cabac decode of qscale diff failed at 43 11 [h264 @ 0x55b60a9ff540] error while decoding MB 43 11, bytestream 115911 [h264 @ 0x55b60b653940] error while decoding MB 55 22, bytestream -30 [h264 @ 0x55b60a911280] error while decoding MB 0 30, bytestream -5 [in#0/rtsp @ 0x55b60a895e00] Error during demuxing: Connection timed out ", "last_exit_at": "2026-07-02T19:25:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T19:25:37+09:00", "pid": 15170, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.102, "name": "ffmpeg", "pid": 15170, "rss_mb": 139.102, "state": "S (sleeping)", "threads": 106}, "restart_count": 63, "restart_count_per_hour": 1, "rtsp_timeout_count": 30, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 64}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 130.823, "sample_count": 2664, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 318514, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T19:25:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.874, "last_frame_pushed_at": "2026-07-02T19:25:34+09:00", "last_frame_read_age_sec": 4.976, "last_frame_read_at": "2026-07-02T19:25:34+09:00", "mjpeg": {"age_sec": 4.86, "exists": true, "mtime": "2026-07-02T19:25:34.810603+09:00", "mtime_epoch": 1782987934.810603, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 162214}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T19:25:39+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1436.215, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 62, "pushed_frames": 62, "read_frames": 61, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.07, "pushed": 2.07, "read": 2.037}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.255, "count": 62, "max_ms": 6.229, "p95_ms": 5.903}, "gstreamer_push": {"avg_ms": 1.296, "count": 62, "max_ms": 1.517, "p95_ms": 1.402}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.393, "count": 62, "max_ms": 71.635, "p95_ms": 66.935}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.159, "count": 62, "max_ms": 87.069, "p95_ms": 84.397}, "yolo11_640_fire_11class": {"avg_ms": 51.457, "count": 62, "max_ms": 72.592, "p95_ms": 66.945}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.364, "count": 62, "max_ms": 88.643, "p95_ms": 84.252}}, "truncated_count": 0}, "processing_total": {"avg_ms": 80.714, "count": 62, "max_ms": 133.698, "p95_ms": 101.836}, "read_interval": {"avg_ms": 410.521, "count": 61, "max_ms": 19206.733, "p95_ms": 120.309}, "triton_wait": {"avg_ms": 65.966, "count": 62, "max_ms": 88.655, "p95_ms": 84.796}, "visualize": {"avg_ms": 3.745, "count": 62, "max_ms": 5.25, "p95_ms": 4.676}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.951, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2724}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5553.609, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x564b95982a80] error while decoding MB 31 27, bytestream -5", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 50, "h264_decode_error_count": 103, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T19:40:36+09:00", "last_error_summary": "[h264 @ 0x564b95982a80] error while decoding MB 31 27, bytestream -5", "last_exit_at": "2026-07-02T19:25:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 898.553, "last_started_at": "2026-07-02T19:25:37+09:00", "pid": 15170, "process": {"exists": false, "pid": 15170}, "restart_count": 63, "restart_count_per_hour": 1, "rtsp_timeout_count": 30, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 64}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.477, "sample_count": 2587, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 326947, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T19:40:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.068, "last_frame_pushed_at": "2026-07-02T19:40:36+09:00", "last_frame_read_age_sec": 0.059, "last_frame_read_at": "2026-07-02T19:40:36+09:00", "mjpeg": {"age_sec": 0.063, "exists": true, "mtime": "2026-07-02T19:40:36.160795+09:00", "mtime_epoch": 1782988836.1607952, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 163692}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T19:40:36+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1440.34, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 213, "pushed_frames": 213, "read_frames": 213, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.128, "pushed": 7.128, "read": 7.128}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.105, "count": 213, "max_ms": 9.587, "p95_ms": 8.169}, "gstreamer_push": {"avg_ms": 1.046, "count": 213, "max_ms": 1.689, "p95_ms": 1.21}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.192, "count": 213, "max_ms": 72.631, "p95_ms": 65.355}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.427, "count": 213, "max_ms": 96.602, "p95_ms": 87.412}, "yolo11_640_fire_11class": {"avg_ms": 50.521, "count": 213, "max_ms": 74.603, "p95_ms": 65.423}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.784, "count": 213, "max_ms": 92.149, "p95_ms": 80.217}}, "truncated_count": 0}, "processing_total": {"avg_ms": 131.146, "count": 213, "max_ms": 179.652, "p95_ms": 163.086}, "read_interval": {"avg_ms": 140.64, "count": 213, "max_ms": 188.49, "p95_ms": 173.113}, "triton_wait": {"avg_ms": 63.726, "count": 213, "max_ms": 96.615, "p95_ms": 90.981}, "visualize": {"avg_ms": 5.787, "count": 213, "max_ms": 7.602, "p95_ms": 6.744}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.881, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2724}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5553.379, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T19:40:36+09:00", "last_error_summary": "[h264 @ 0x564b95982a80] error while decoding MB 31 27, bytestream -5", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 50, "h264_decode_error_count": 103, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T19:40:36+09:00", "last_error_summary": "[h264 @ 0x564b95982a80] error while decoding MB 31 27, bytestream -5", "last_exit_at": "2026-07-02T19:40:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 898.606, "last_started_at": "2026-07-02T19:25:37+09:00", "pid": 15170, "process": {"exists": false, "pid": 15170}, "restart_count": 63, "restart_count_per_hour": 1, "rtsp_timeout_count": 30, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 64}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.477, "sample_count": 2588, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 326948, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T19:40:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T19:40:36+09:00", "last_frame_read_age_sec": 0.112, "last_frame_read_at": "2026-07-02T19:40:36+09:00", "mjpeg": {"age_sec": 0.115, "exists": true, "mtime": "2026-07-02T19:40:36.160795+09:00", "mtime_epoch": 1782988836.1607952, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 163692}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T19:40:36+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1440.34, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 214, "pushed_frames": 214, "read_frames": 213, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.149, "pushed": 7.149, "read": 7.116}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.131, "count": 214, "max_ms": 12.788, "p95_ms": 8.224}, "gstreamer_push": {"avg_ms": 1.045, "count": 214, "max_ms": 1.689, "p95_ms": 1.21}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.164, "count": 214, "max_ms": 72.631, "p95_ms": 65.355}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.385, "count": 214, "max_ms": 96.602, "p95_ms": 87.412}, "yolo11_640_fire_11class": {"avg_ms": 50.493, "count": 214, "max_ms": 74.603, "p95_ms": 65.423}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.754, "count": 214, "max_ms": 92.149, "p95_ms": 80.217}}, "truncated_count": 0}, "processing_total": {"avg_ms": 131.03, "count": 214, "max_ms": 179.652, "p95_ms": 163.086}, "read_interval": {"avg_ms": 140.64, "count": 213, "max_ms": 188.49, "p95_ms": 173.113}, "triton_wait": {"avg_ms": 63.677, "count": 214, "max_ms": 96.615, "p95_ms": 90.981}, "visualize": {"avg_ms": 5.802, "count": 214, "max_ms": 9.017, "p95_ms": 6.771}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.933, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2724}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5559.371, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 50, "h264_decode_error_count": 103, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T19:40:36+09:00", "last_error_summary": "[h264 @ 0x564b95982a80] error while decoding MB 31 27, bytestream -5", "last_exit_at": "2026-07-02T19:40:36+09:00", "last_returncode": 0, "last_stable_duration_sec": 898.611, "last_started_at": "2026-07-02T19:25:37+09:00", "pid": 15170, "process": {"exists": false, "pid": 15170}, "restart_count": 63, "restart_count_per_hour": 1, "rtsp_timeout_count": 30, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": false, "start_count": 64}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.477, "sample_count": 2588, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 326948, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T19:40:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-02T19:40:36+09:00", "last_frame_read_age_sec": 0.118, "last_frame_read_at": "2026-07-02T19:40:36+09:00", "mjpeg": {"age_sec": 0.12, "exists": true, "mtime": "2026-07-02T19:40:36.160795+09:00", "mtime_epoch": 1782988836.1607952, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 163692}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T19:40:36+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1440.34, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 214, "pushed_frames": 214, "read_frames": 213, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.148, "pushed": 7.148, "read": 7.115}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.131, "count": 214, "max_ms": 12.788, "p95_ms": 8.224}, "gstreamer_push": {"avg_ms": 1.045, "count": 214, "max_ms": 1.689, "p95_ms": 1.21}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.164, "count": 214, "max_ms": 72.631, "p95_ms": 65.355}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.385, "count": 214, "max_ms": 96.602, "p95_ms": 87.412}, "yolo11_640_fire_11class": {"avg_ms": 50.493, "count": 214, "max_ms": 74.603, "p95_ms": 65.423}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.754, "count": 214, "max_ms": 92.149, "p95_ms": 80.217}}, "truncated_count": 0}, "processing_total": {"avg_ms": 131.03, "count": 214, "max_ms": 179.652, "p95_ms": 163.086}, "read_interval": {"avg_ms": 140.64, "count": 213, "max_ms": 188.49, "p95_ms": 173.113}, "triton_wait": {"avg_ms": 63.677, "count": 214, "max_ms": 96.615, "p95_ms": 90.981}, "visualize": {"avg_ms": 5.802, "count": 214, "max_ms": 9.017, "p95_ms": 6.771}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.938, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2724}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5661.043, "limit_mb": 8796093022207.996}, "detail": {"pid": 15277, "restart_count": 64, "start_count": 65}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 50, "h264_decode_error_count": 103, "h264_decode_error_count_per_hour": 6, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T19:40:36+09:00", "last_error_summary": "[h264 @ 0x564b95982a80] error while decoding MB 31 27, bytestream -5", "last_exit_at": "2026-07-02T19:40:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T19:40:38+09:00", "pid": 15277, "process": {"exists": true, "fd_count": 5, "hwm_mb": 135.93, "name": "ffmpeg", "pid": 15277, "rss_mb": 135.93, "state": "S (sleeping)", "threads": 106}, "restart_count": 64, "restart_count_per_hour": 2, "rtsp_timeout_count": 30, "rtsp_timeout_count_per_hour": 2, "rtsp_tolerant_mode": false, "running": true, "start_count": 65}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 155.568, "sample_count": 2538, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 326948, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T19:40:36+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": []}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.459, "last_frame_pushed_at": "2026-07-02T19:40:36+09:00", "last_frame_read_age_sec": 4.567, "last_frame_read_at": "2026-07-02T19:40:36+09:00", "mjpeg": {"age_sec": 4.446, "exists": true, "mtime": "2026-07-02T19:40:36.284797+09:00", "mtime_epoch": 1782988836.2847972, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 163320}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T19:40:40+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1440.34, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_unstable_after_restart_critical"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 180, "pushed_frames": 180, "read_frames": 179, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.003, "pushed": 6.003, "read": 5.97}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.232, "count": 179, "max_ms": 12.788, "p95_ms": 8.259}, "gstreamer_push": {"avg_ms": 1.032, "count": 180, "max_ms": 1.689, "p95_ms": 1.208}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.764, "count": 179, "max_ms": 72.631, "p95_ms": 66.022}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.822, "count": 179, "max_ms": 96.602, "p95_ms": 90.968}, "yolo11_640_fire_11class": {"avg_ms": 50.008, "count": 179, "max_ms": 74.603, "p95_ms": 65.491}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.297, "count": 179, "max_ms": 92.149, "p95_ms": 80.659}}, "truncated_count": 0}, "processing_total": {"avg_ms": 133.215, "count": 180, "max_ms": 179.652, "p95_ms": 165.063}, "read_interval": {"avg_ms": 142.759, "count": 179, "max_ms": 188.49, "p95_ms": 174.512}, "triton_wait": {"avg_ms": 63.121, "count": 179, "max_ms": 96.615, "p95_ms": 91.523}, "visualize": {"avg_ms": 5.924, "count": 179, "max_ms": 9.017, "p95_ms": 6.875}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.984, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2747}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5567.066, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 3, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55a86a92d5c0] left block unavailable for requested intra mode [h264 @ 0x55a86a92d5c0] error while decoding MB 0 13, bytestream 767 [h264 @ 0x55a86a8c0600] cabac decode of qscale diff failed at 55 15 [h264 @ 0x55a86a8c0600] error while decoding MB 55 15, bytestream 7671 [in#0/rtsp @ 0x55a86a7bae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a86a7bae00] Error retrieving a packet from demuxer: Connection timed out", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 51, "h264_decode_error_count": 106, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T19:46:05+09:00", "last_error_summary": "[h264 @ 0x55a86a92d5c0] left block unavailable for requested intra mode [h264 @ 0x55a86a92d5c0] error while decoding MB 0 13, bytestream 767 [h264 @ 0x55a86a8c0600] cabac decode of qscale diff failed at 55 15 [h264 @ 0x55a86a8c0600] error while decoding MB 55 15, bytestream 7671 [in#0/rtsp @ 0x55a86a7bae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a86a7bae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T19:40:38+09:00", "last_returncode": 0, "last_stable_duration_sec": 327.101, "last_started_at": "2026-07-02T19:40:38+09:00", "pid": 15277, "process": {"exists": false, "pid": 15277}, "restart_count": 64, "restart_count_per_hour": 2, "rtsp_timeout_count": 32, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 65}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.392, "sample_count": 2545, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 329692, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T19:46:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.063, "last_frame_pushed_at": "2026-07-02T19:46:05+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-02T19:46:05+09:00", "mjpeg": {"age_sec": 0.044, "exists": true, "mtime": "2026-07-02T19:46:05.786250+09:00", "mtime_epoch": 1782989165.7862504, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164245}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T19:46:05+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1441.629, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 112, "pushed_frames": 112, "read_frames": 112, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.734, "pushed": 3.734, "read": 3.734}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.919, "count": 111, "max_ms": 10.664, "p95_ms": 7.184}, "gstreamer_push": {"avg_ms": 1.178, "count": 112, "max_ms": 1.657, "p95_ms": 1.396}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.781, "count": 111, "max_ms": 70.983, "p95_ms": 62.525}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.915, "count": 111, "max_ms": 103.786, "p95_ms": 84.99}, "yolo11_640_fire_11class": {"avg_ms": 47.219, "count": 111, "max_ms": 65.595, "p95_ms": 59.815}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.241, "count": 111, "max_ms": 87.822, "p95_ms": 78.676}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.507, "count": 112, "max_ms": 170.185, "p95_ms": 142.773}, "read_interval": {"avg_ms": 267.98, "count": 112, "max_ms": 10119.924, "p95_ms": 159.892}, "triton_wait": {"avg_ms": 62.38, "count": 111, "max_ms": 103.798, "p95_ms": 85.355}, "visualize": {"avg_ms": 4.028, "count": 111, "max_ms": 7.295, "p95_ms": 5.24}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.997, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2747}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5568.957, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T19:46:05+09:00", "last_error_summary": "[h264 @ 0x55a86a92d5c0] left block unavailable for requested intra mode [h264 @ 0x55a86a92d5c0] error while decoding MB 0 13, bytestream 767 [h264 @ 0x55a86a8c0600] cabac decode of qscale diff failed at 55 15 [h264 @ 0x55a86a8c0600] error while decoding MB 55 15, bytestream 7671 [in#0/rtsp @ 0x55a86a7bae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a86a7bae00] Error retrieving a packet from demuxer: Connection timed out", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 51, "h264_decode_error_count": 106, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T19:46:05+09:00", "last_error_summary": "[h264 @ 0x55a86a92d5c0] left block unavailable for requested intra mode [h264 @ 0x55a86a92d5c0] error while decoding MB 0 13, bytestream 767 [h264 @ 0x55a86a8c0600] cabac decode of qscale diff failed at 55 15 [h264 @ 0x55a86a8c0600] error while decoding MB 55 15, bytestream 7671 [in#0/rtsp @ 0x55a86a7bae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a86a7bae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T19:46:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 327.171, "last_started_at": "2026-07-02T19:40:38+09:00", "pid": 15277, "process": {"exists": false, "pid": 15277}, "restart_count": 64, "restart_count_per_hour": 2, "rtsp_timeout_count": 32, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 65}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.392, "sample_count": 2546, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 329693, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T19:46:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T19:46:05+09:00", "last_frame_read_age_sec": 0.124, "last_frame_read_at": "2026-07-02T19:46:05+09:00", "mjpeg": {"age_sec": 0.114, "exists": true, "mtime": "2026-07-02T19:46:05.786250+09:00", "mtime_epoch": 1782989165.7862504, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164245}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T19:46:05+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1441.629, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 112, "pushed_frames": 112, "read_frames": 111, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.737, "pushed": 3.737, "read": 3.703}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.981, "count": 112, "max_ms": 11.858, "p95_ms": 7.31}, "gstreamer_push": {"avg_ms": 1.177, "count": 112, "max_ms": 1.657, "p95_ms": 1.396}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.754, "count": 112, "max_ms": 70.983, "p95_ms": 62.525}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.876, "count": 112, "max_ms": 103.786, "p95_ms": 84.99}, "yolo11_640_fire_11class": {"avg_ms": 47.211, "count": 112, "max_ms": 65.595, "p95_ms": 59.815}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.188, "count": 112, "max_ms": 87.822, "p95_ms": 78.676}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.925, "count": 112, "max_ms": 170.185, "p95_ms": 142.773}, "read_interval": {"avg_ms": 269.65, "count": 111, "max_ms": 10119.924, "p95_ms": 159.892}, "triton_wait": {"avg_ms": 62.327, "count": 112, "max_ms": 103.798, "p95_ms": 85.355}, "visualize": {"avg_ms": 4.041, "count": 112, "max_ms": 7.295, "p95_ms": 5.413}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.974, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2747}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5568.926, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 51, "h264_decode_error_count": 106, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T19:46:05+09:00", "last_error_summary": "[h264 @ 0x55a86a92d5c0] left block unavailable for requested intra mode [h264 @ 0x55a86a92d5c0] error while decoding MB 0 13, bytestream 767 [h264 @ 0x55a86a8c0600] cabac decode of qscale diff failed at 55 15 [h264 @ 0x55a86a8c0600] error while decoding MB 55 15, bytestream 7671 [in#0/rtsp @ 0x55a86a7bae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a86a7bae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T19:46:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 327.177, "last_started_at": "2026-07-02T19:40:38+09:00", "pid": 15277, "process": {"exists": false, "pid": 15277}, "restart_count": 64, "restart_count_per_hour": 2, "rtsp_timeout_count": 32, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": false, "start_count": 65}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.392, "sample_count": 2546, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 329693, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T19:46:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-02T19:46:05+09:00", "last_frame_read_age_sec": 0.129, "last_frame_read_at": "2026-07-02T19:46:05+09:00", "mjpeg": {"age_sec": 0.12, "exists": true, "mtime": "2026-07-02T19:46:05.786250+09:00", "mtime_epoch": 1782989165.7862504, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164245}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T19:46:05+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1441.629, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_unstable_after_restart_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 112, "pushed_frames": 112, "read_frames": 111, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 3.736, "pushed": 3.736, "read": 3.703}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 4.981, "count": 112, "max_ms": 11.858, "p95_ms": 7.31}, "gstreamer_push": {"avg_ms": 1.177, "count": 112, "max_ms": 1.657, "p95_ms": 1.396}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 48.754, "count": 112, "max_ms": 70.983, "p95_ms": 62.525}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.876, "count": 112, "max_ms": 103.786, "p95_ms": 84.99}, "yolo11_640_fire_11class": {"avg_ms": 47.211, "count": 112, "max_ms": 65.595, "p95_ms": 59.815}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.188, "count": 112, "max_ms": 87.822, "p95_ms": 78.676}}, "truncated_count": 0}, "processing_total": {"avg_ms": 89.925, "count": 112, "max_ms": 170.185, "p95_ms": 142.773}, "read_interval": {"avg_ms": 269.65, "count": 111, "max_ms": 10119.924, "p95_ms": 159.892}, "triton_wait": {"avg_ms": 62.327, "count": 112, "max_ms": 103.798, "p95_ms": 85.355}, "visualize": {"avg_ms": 4.041, "count": 112, "max_ms": 7.295, "p95_ms": 5.413}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.979, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2747}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5671.68, "limit_mb": 8796093022207.996}, "detail": {"pid": 15384, "restart_count": 65, "start_count": 66}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 51, "h264_decode_error_count": 106, "h264_decode_error_count_per_hour": 9, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T19:46:05+09:00", "last_error_summary": "[h264 @ 0x55a86a92d5c0] left block unavailable for requested intra mode [h264 @ 0x55a86a92d5c0] error while decoding MB 0 13, bytestream 767 [h264 @ 0x55a86a8c0600] cabac decode of qscale diff failed at 55 15 [h264 @ 0x55a86a8c0600] error while decoding MB 55 15, bytestream 7671 [in#0/rtsp @ 0x55a86a7bae00] Error during demuxing: Connection timed out [in#0/rtsp @ 0x55a86a7bae00] Error retrieving a packet from demuxer: Connection timed out", "last_exit_at": "2026-07-02T19:46:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T19:46:08+09:00", "pid": 15384, "process": {"exists": true, "fd_count": 5, "hwm_mb": 136.488, "name": "ffmpeg", "pid": 15384, "rss_mb": 136.488, "state": "S (sleeping)", "threads": 106}, "restart_count": 65, "restart_count_per_hour": 3, "rtsp_timeout_count": 32, "rtsp_timeout_count_per_hour": 4, "rtsp_tolerant_mode": false, "running": true, "start_count": 66}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 145.392, "sample_count": 2509, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 329693, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T19:46:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.777, "last_frame_pushed_at": "2026-07-02T19:46:05+09:00", "last_frame_read_age_sec": 4.896, "last_frame_read_at": "2026-07-02T19:46:05+09:00", "mjpeg": {"age_sec": 4.751, "exists": true, "mtime": "2026-07-02T19:46:05.922253+09:00", "mtime_epoch": 1782989165.9222527, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164285}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T19:46:10+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1441.629, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 55, "pushed_frames": 55, "read_frames": 54, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.836, "pushed": 1.836, "read": 1.803}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 5.921, "count": 55, "max_ms": 11.858, "p95_ms": 8.155}, "gstreamer_push": {"avg_ms": 1.125, "count": 55, "max_ms": 1.657, "p95_ms": 1.379}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 49.852, "count": 55, "max_ms": 70.983, "p95_ms": 67.322}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.35, "count": 55, "max_ms": 103.786, "p95_ms": 98.764}, "yolo11_640_fire_11class": {"avg_ms": 47.016, "count": 55, "max_ms": 65.595, "p95_ms": 62.586}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.685, "count": 55, "max_ms": 87.822, "p95_ms": 80.741}}, "truncated_count": 0}, "processing_total": {"avg_ms": 109.469, "count": 55, "max_ms": 170.185, "p95_ms": 154.203}, "read_interval": {"avg_ms": 465.393, "count": 54, "max_ms": 10119.924, "p95_ms": 180.662}, "triton_wait": {"avg_ms": 66.08, "count": 55, "max_ms": 103.798, "p95_ms": 98.777}, "visualize": {"avg_ms": 4.614, "count": 55, "max_ms": 7.295, "p95_ms": 6.26}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.954, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2767}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5570.688, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 4, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x560083a31b80] cabac decode of qscale diff failed at 27 32 [h264 @ 0x560083a31b80] error while decoding MB 27 32, bytestream 111613 [h264 @ 0x56008461a6c0] left block unavailable for requested intra mode [h264 @ 0x56008461a6c0] error while decoding MB 0 20, bytestream 100407 [h264 @ 0x560083a31b80] left block unavailable for requested intra mode [h264 @ 0x560083a31b80] error while decoding MB 0 21, bytestream 4580 [in#0/rtsp @ 0x560083922e00] Error during demuxing: Connection timed out ", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 52, "h264_decode_error_count": 110, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T19:51:05+09:00", "last_error_summary": "[h264 @ 0x560083a31b80] cabac decode of qscale diff failed at 27 32 [h264 @ 0x560083a31b80] error while decoding MB 27 32, bytestream 111613 [h264 @ 0x56008461a6c0] left block unavailable for requested intra mode [h264 @ 0x56008461a6c0] error while decoding MB 0 20, bytestream 100407 [h264 @ 0x560083a31b80] left block unavailable for requested intra mode [h264 @ 0x560083a31b80] error while decoding MB 0 21, bytestream 4580 [in#0/rtsp @ 0x560083922e00] Error during demuxing: Connection timed out ", "last_exit_at": "2026-07-02T19:46:08+09:00", "last_returncode": 0, "last_stable_duration_sec": 296.945, "last_started_at": "2026-07-02T19:46:08+09:00", "pid": 15384, "process": {"exists": false, "pid": 15384}, "restart_count": 65, "restart_count_per_hour": 3, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 66}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 150.845, "sample_count": 2437, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 332127, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T19:51:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.064, "last_frame_pushed_at": "2026-07-02T19:51:05+09:00", "last_frame_read_age_sec": 0.058, "last_frame_read_at": "2026-07-02T19:51:05+09:00", "mjpeg": {"age_sec": 0.048, "exists": true, "mtime": "2026-07-02T19:51:05.571253+09:00", "mtime_epoch": 1782989465.5712528, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164385}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T19:51:05+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1442.66, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 83, "pushed_frames": 83, "read_frames": 83, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.773, "pushed": 2.773, "read": 2.773}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.571, "count": 83, "max_ms": 9.604, "p95_ms": 8.441}, "gstreamer_push": {"avg_ms": 1.12, "count": 83, "max_ms": 1.485, "p95_ms": 1.416}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.705, "count": 83, "max_ms": 74.105, "p95_ms": 63.965}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.732, "count": 83, "max_ms": 92.828, "p95_ms": 84.971}, "yolo11_640_fire_11class": {"avg_ms": 50.112, "count": 83, "max_ms": 74.095, "p95_ms": 63.324}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.519, "count": 83, "max_ms": 95.711, "p95_ms": 78.345}}, "truncated_count": 0}, "processing_total": {"avg_ms": 120.714, "count": 83, "max_ms": 173.654, "p95_ms": 155.224}, "read_interval": {"avg_ms": 361.63, "count": 83, "max_ms": 10132.76, "p95_ms": 167.5}, "triton_wait": {"avg_ms": 65.44, "count": 83, "max_ms": 95.723, "p95_ms": 87.114}, "visualize": {"avg_ms": 5.374, "count": 83, "max_ms": 7.808, "p95_ms": 6.772}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.932, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2767}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5570.164, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T19:51:05+09:00", "last_error_summary": "[h264 @ 0x560083a31b80] cabac decode of qscale diff failed at 27 32 [h264 @ 0x560083a31b80] error while decoding MB 27 32, bytestream 111613 [h264 @ 0x56008461a6c0] left block unavailable for requested intra mode [h264 @ 0x56008461a6c0] error while decoding MB 0 20, bytestream 100407 [h264 @ 0x560083a31b80] left block unavailable for requested intra mode [h264 @ 0x560083a31b80] error while decoding MB 0 21, bytestream 4580 [in#0/rtsp @ 0x560083922e00] Error during demuxing: Connection timed out ", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 52, "h264_decode_error_count": 110, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T19:51:05+09:00", "last_error_summary": "[h264 @ 0x560083a31b80] cabac decode of qscale diff failed at 27 32 [h264 @ 0x560083a31b80] error while decoding MB 27 32, bytestream 111613 [h264 @ 0x56008461a6c0] left block unavailable for requested intra mode [h264 @ 0x56008461a6c0] error while decoding MB 0 20, bytestream 100407 [h264 @ 0x560083a31b80] left block unavailable for requested intra mode [h264 @ 0x560083a31b80] error while decoding MB 0 21, bytestream 4580 [in#0/rtsp @ 0x560083922e00] Error during demuxing: Connection timed out ", "last_exit_at": "2026-07-02T19:51:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 297.01, "last_started_at": "2026-07-02T19:46:08+09:00", "pid": 15384, "process": {"exists": false, "pid": 15384}, "restart_count": 65, "restart_count_per_hour": 3, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 66}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 150.845, "sample_count": 2437, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 332128, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T19:51:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T19:51:05+09:00", "last_frame_read_age_sec": 0.123, "last_frame_read_at": "2026-07-02T19:51:05+09:00", "mjpeg": {"age_sec": 0.111, "exists": true, "mtime": "2026-07-02T19:51:05.571253+09:00", "mtime_epoch": 1782989465.5712528, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164385}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T19:51:05+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1442.66, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 84, "pushed_frames": 84, "read_frames": 83, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.801, "pushed": 2.801, "read": 2.767}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.641, "count": 83, "max_ms": 13.33, "p95_ms": 8.473}, "gstreamer_push": {"avg_ms": 1.121, "count": 84, "max_ms": 1.485, "p95_ms": 1.416}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.625, "count": 83, "max_ms": 74.105, "p95_ms": 63.965}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.591, "count": 83, "max_ms": 92.828, "p95_ms": 84.971}, "yolo11_640_fire_11class": {"avg_ms": 50.049, "count": 83, "max_ms": 74.095, "p95_ms": 63.324}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.418, "count": 83, "max_ms": 95.711, "p95_ms": 78.345}}, "truncated_count": 0}, "processing_total": {"avg_ms": 120.662, "count": 84, "max_ms": 173.654, "p95_ms": 155.224}, "read_interval": {"avg_ms": 361.63, "count": 83, "max_ms": 10132.76, "p95_ms": 167.5}, "triton_wait": {"avg_ms": 65.308, "count": 83, "max_ms": 95.723, "p95_ms": 87.114}, "visualize": {"avg_ms": 5.384, "count": 83, "max_ms": 7.808, "p95_ms": 6.96}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.994, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2767}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5570.156, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 52, "h264_decode_error_count": 110, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T19:51:05+09:00", "last_error_summary": "[h264 @ 0x560083a31b80] cabac decode of qscale diff failed at 27 32 [h264 @ 0x560083a31b80] error while decoding MB 27 32, bytestream 111613 [h264 @ 0x56008461a6c0] left block unavailable for requested intra mode [h264 @ 0x56008461a6c0] error while decoding MB 0 20, bytestream 100407 [h264 @ 0x560083a31b80] left block unavailable for requested intra mode [h264 @ 0x560083a31b80] error while decoding MB 0 21, bytestream 4580 [in#0/rtsp @ 0x560083922e00] Error during demuxing: Connection timed out ", "last_exit_at": "2026-07-02T19:51:05+09:00", "last_returncode": 0, "last_stable_duration_sec": 297.016, "last_started_at": "2026-07-02T19:46:08+09:00", "pid": 15384, "process": {"exists": false, "pid": 15384}, "restart_count": 65, "restart_count_per_hour": 3, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 66}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 150.845, "sample_count": 2437, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 332128, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T19:51:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-02T19:51:05+09:00", "last_frame_read_age_sec": 0.129, "last_frame_read_at": "2026-07-02T19:51:05+09:00", "mjpeg": {"age_sec": 0.116, "exists": true, "mtime": "2026-07-02T19:51:05.571253+09:00", "mtime_epoch": 1782989465.5712528, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164385}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T19:51:05+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1442.66, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 84, "pushed_frames": 84, "read_frames": 83, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.8, "pushed": 2.8, "read": 2.767}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.641, "count": 83, "max_ms": 13.33, "p95_ms": 8.473}, "gstreamer_push": {"avg_ms": 1.121, "count": 84, "max_ms": 1.485, "p95_ms": 1.416}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.625, "count": 83, "max_ms": 74.105, "p95_ms": 63.965}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 63.591, "count": 83, "max_ms": 92.828, "p95_ms": 84.971}, "yolo11_640_fire_11class": {"avg_ms": 50.049, "count": 83, "max_ms": 74.095, "p95_ms": 63.324}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.418, "count": 83, "max_ms": 95.711, "p95_ms": 78.345}}, "truncated_count": 0}, "processing_total": {"avg_ms": 120.662, "count": 84, "max_ms": 173.654, "p95_ms": 155.224}, "read_interval": {"avg_ms": 361.63, "count": 83, "max_ms": 10132.76, "p95_ms": 167.5}, "triton_wait": {"avg_ms": 65.308, "count": 83, "max_ms": 95.723, "p95_ms": 87.114}, "visualize": {"avg_ms": 5.384, "count": 83, "max_ms": 7.808, "p95_ms": 6.96}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 30.0, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2767}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5681.453, "limit_mb": 8796093022207.996}, "detail": {"pid": 15491, "restart_count": 66, "start_count": 67}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 52, "h264_decode_error_count": 110, "h264_decode_error_count_per_hour": 13, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T19:51:05+09:00", "last_error_summary": "[h264 @ 0x560083a31b80] cabac decode of qscale diff failed at 27 32 [h264 @ 0x560083a31b80] error while decoding MB 27 32, bytestream 111613 [h264 @ 0x56008461a6c0] left block unavailable for requested intra mode [h264 @ 0x56008461a6c0] error while decoding MB 0 20, bytestream 100407 [h264 @ 0x560083a31b80] left block unavailable for requested intra mode [h264 @ 0x560083a31b80] error while decoding MB 0 21, bytestream 4580 [in#0/rtsp @ 0x560083922e00] Error during demuxing: Connection timed out ", "last_exit_at": "2026-07-02T19:51:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T19:51:07+09:00", "pid": 15491, "process": {"exists": true, "fd_count": 5, "hwm_mb": 141.652, "name": "ffmpeg", "pid": 15491, "rss_mb": 141.652, "state": "S (sleeping)", "threads": 106}, "restart_count": 66, "restart_count_per_hour": 4, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 67}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 150.845, "sample_count": 2435, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 332128, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T19:51:05+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.166, "last_frame_pushed_at": "2026-07-02T19:51:05+09:00", "last_frame_read_age_sec": 4.285, "last_frame_read_at": "2026-07-02T19:51:05+09:00", "mjpeg": {"age_sec": 4.148, "exists": true, "mtime": "2026-07-02T19:51:05.695255+09:00", "mtime_epoch": 1782989465.695255, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164584}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T19:51:09+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1442.66, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 56, "pushed_frames": 56, "read_frames": 55, "select_timeouts": 19, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.873, "pushed": 1.873, "read": 1.839}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.191, "count": 56, "max_ms": 13.33, "p95_ms": 8.441}, "gstreamer_push": {"avg_ms": 1.167, "count": 56, "max_ms": 1.485, "p95_ms": 1.431}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.071, "count": 56, "max_ms": 66.711, "p95_ms": 65.048}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 64.298, "count": 56, "max_ms": 92.828, "p95_ms": 87.941}, "yolo11_640_fire_11class": {"avg_ms": 49.823, "count": 56, "max_ms": 68.555, "p95_ms": 63.977}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.353, "count": 56, "max_ms": 84.864, "p95_ms": 79.381}}, "truncated_count": 0}, "processing_total": {"avg_ms": 109.434, "count": 56, "max_ms": 165.754, "p95_ms": 152.065}, "read_interval": {"avg_ms": 468.336, "count": 55, "max_ms": 10132.76, "p95_ms": 175.117}, "triton_wait": {"avg_ms": 65.999, "count": 56, "max_ms": 92.84, "p95_ms": 87.954}, "visualize": {"avg_ms": 4.895, "count": 56, "max_ms": 7.808, "p95_ms": 6.772}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.901, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2787}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5579.582, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x557945da5980] error while decoding MB 46 9, bytestream -6", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 53, "h264_decode_error_count": 111, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T19:55:59+09:00", "last_error_summary": "[h264 @ 0x557945da5980] error while decoding MB 46 9, bytestream -6", "last_exit_at": "2026-07-02T19:51:07+09:00", "last_returncode": 0, "last_stable_duration_sec": 291.582, "last_started_at": "2026-07-02T19:51:07+09:00", "pid": 15491, "process": {"exists": false, "pid": 15491}, "restart_count": 66, "restart_count_per_hour": 4, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 67}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 149.265, "sample_count": 2402, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 334522, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T19:55:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.059, "last_frame_pushed_at": "2026-07-02T19:55:59+09:00", "last_frame_read_age_sec": 0.051, "last_frame_read_at": "2026-07-02T19:55:59+09:00", "mjpeg": {"age_sec": 0.041, "exists": true, "mtime": "2026-07-02T19:55:59.384176+09:00", "mtime_epoch": 1782989759.3841765, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164472}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T19:55:59+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1443.949, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 257, "pushed_frames": 257, "read_frames": 257, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.606, "pushed": 8.606, "read": 8.606}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.762, "count": 257, "max_ms": 11.053, "p95_ms": 7.751}, "gstreamer_push": {"avg_ms": 1.17, "count": 257, "max_ms": 1.899, "p95_ms": 1.406}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.033, "count": 257, "max_ms": 76.252, "p95_ms": 66.853}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.818, "count": 257, "max_ms": 96.721, "p95_ms": 89.27}, "yolo11_640_fire_11class": {"avg_ms": 50.291, "count": 257, "max_ms": 75.464, "p95_ms": 65.882}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.412, "count": 257, "max_ms": 92.221, "p95_ms": 82.599}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.448, "count": 257, "max_ms": 172.956, "p95_ms": 147.441}, "read_interval": {"avg_ms": 116.565, "count": 257, "max_ms": 183.481, "p95_ms": 157.58}, "triton_wait": {"avg_ms": 65.711, "count": 257, "max_ms": 96.734, "p95_ms": 90.245}, "visualize": {"avg_ms": 4.674, "count": 257, "max_ms": 7.281, "p95_ms": 6.344}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.864, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2787}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5579.176, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T19:55:59+09:00", "last_error_summary": "[h264 @ 0x557945da5980] error while decoding MB 46 9, bytestream -6", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 53, "h264_decode_error_count": 111, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T19:55:59+09:00", "last_error_summary": "[h264 @ 0x557945da5980] error while decoding MB 46 9, bytestream -6", "last_exit_at": "2026-07-02T19:55:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 291.684, "last_started_at": "2026-07-02T19:51:07+09:00", "pid": 15491, "process": {"exists": false, "pid": 15491}, "restart_count": 66, "restart_count_per_hour": 4, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 67}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 149.265, "sample_count": 2403, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 334523, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T19:55:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T19:55:59+09:00", "last_frame_read_age_sec": 0.152, "last_frame_read_at": "2026-07-02T19:55:59+09:00", "mjpeg": {"age_sec": 0.142, "exists": true, "mtime": "2026-07-02T19:55:59.384176+09:00", "mtime_epoch": 1782989759.3841765, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164472}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T19:55:59+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1443.949, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 258, "pushed_frames": 258, "read_frames": 257, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.61, "pushed": 8.61, "read": 8.577}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.763, "count": 257, "max_ms": 11.053, "p95_ms": 7.751}, "gstreamer_push": {"avg_ms": 1.17, "count": 258, "max_ms": 1.899, "p95_ms": 1.406}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.079, "count": 257, "max_ms": 76.252, "p95_ms": 67.374}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.68, "count": 257, "max_ms": 96.721, "p95_ms": 88.985}, "yolo11_640_fire_11class": {"avg_ms": 50.334, "count": 257, "max_ms": 75.464, "p95_ms": 66.179}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.488, "count": 257, "max_ms": 92.221, "p95_ms": 83.357}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.599, "count": 258, "max_ms": 172.956, "p95_ms": 147.441}, "read_interval": {"avg_ms": 116.565, "count": 257, "max_ms": 183.481, "p95_ms": 157.58}, "triton_wait": {"avg_ms": 65.695, "count": 257, "max_ms": 96.734, "p95_ms": 89.282}, "visualize": {"avg_ms": 4.675, "count": 257, "max_ms": 7.281, "p95_ms": 6.344}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.965, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2787}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5579.168, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 53, "h264_decode_error_count": 111, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T19:55:59+09:00", "last_error_summary": "[h264 @ 0x557945da5980] error while decoding MB 46 9, bytestream -6", "last_exit_at": "2026-07-02T19:55:59+09:00", "last_returncode": 0, "last_stable_duration_sec": 291.69, "last_started_at": "2026-07-02T19:51:07+09:00", "pid": 15491, "process": {"exists": false, "pid": 15491}, "restart_count": 66, "restart_count_per_hour": 4, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 67}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 149.265, "sample_count": 2403, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 334523, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T19:55:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.012, "last_frame_pushed_at": "2026-07-02T19:55:59+09:00", "last_frame_read_age_sec": 0.158, "last_frame_read_at": "2026-07-02T19:55:59+09:00", "mjpeg": {"age_sec": 0.148, "exists": true, "mtime": "2026-07-02T19:55:59.384176+09:00", "mtime_epoch": 1782989759.3841765, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164472}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T19:55:59+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1443.949, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 258, "pushed_frames": 258, "read_frames": 257, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.608, "pushed": 8.608, "read": 8.575}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.763, "count": 257, "max_ms": 11.053, "p95_ms": 7.751}, "gstreamer_push": {"avg_ms": 1.17, "count": 258, "max_ms": 1.899, "p95_ms": 1.406}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.079, "count": 257, "max_ms": 76.252, "p95_ms": 67.374}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.68, "count": 257, "max_ms": 96.721, "p95_ms": 88.985}, "yolo11_640_fire_11class": {"avg_ms": 50.334, "count": 257, "max_ms": 75.464, "p95_ms": 66.179}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.488, "count": 257, "max_ms": 92.221, "p95_ms": 83.357}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.599, "count": 258, "max_ms": 172.956, "p95_ms": 147.441}, "read_interval": {"avg_ms": 116.565, "count": 257, "max_ms": 183.481, "p95_ms": 157.58}, "triton_wait": {"avg_ms": 65.695, "count": 257, "max_ms": 96.734, "p95_ms": 89.282}, "visualize": {"avg_ms": 4.675, "count": 257, "max_ms": 7.281, "p95_ms": 6.344}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.971, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2787}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5689.047, "limit_mb": 8796093022207.996}, "detail": {"pid": 15598, "restart_count": 67, "start_count": 68}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 53, "h264_decode_error_count": 111, "h264_decode_error_count_per_hour": 14, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T19:55:59+09:00", "last_error_summary": "[h264 @ 0x557945da5980] error while decoding MB 46 9, bytestream -6", "last_exit_at": "2026-07-02T19:56:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T19:56:02+09:00", "pid": 15598, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.102, "name": "ffmpeg", "pid": 15598, "rss_mb": 139.102, "state": "S (sleeping)", "threads": 106}, "restart_count": 67, "restart_count_per_hour": 5, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 68}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 149.265, "sample_count": 2403, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 334523, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T19:55:59+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.512, "last_frame_pushed_at": "2026-07-02T19:55:59+09:00", "last_frame_read_age_sec": 4.658, "last_frame_read_at": "2026-07-02T19:55:59+09:00", "mjpeg": {"age_sec": 4.496, "exists": true, "mtime": "2026-07-02T19:55:59.536179+09:00", "mtime_epoch": 1782989759.536179, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164652}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T19:56:04+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1443.949, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_unstable_after_restart_critical", "ffmpeg_churn_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 216, "pushed_frames": 216, "read_frames": 215, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.205, "pushed": 7.205, "read": 7.171}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.911, "count": 215, "max_ms": 11.053, "p95_ms": 8.056}, "gstreamer_push": {"avg_ms": 1.16, "count": 216, "max_ms": 1.899, "p95_ms": 1.406}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.741, "count": 215, "max_ms": 75.596, "p95_ms": 66.853}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.349, "count": 215, "max_ms": 96.721, "p95_ms": 88.985}, "yolo11_640_fire_11class": {"avg_ms": 49.999, "count": 215, "max_ms": 74.856, "p95_ms": 65.882}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.07, "count": 215, "max_ms": 91.086, "p95_ms": 81.904}}, "truncated_count": 0}, "processing_total": {"avg_ms": 107.561, "count": 216, "max_ms": 172.956, "p95_ms": 148.534}, "read_interval": {"avg_ms": 118.298, "count": 215, "max_ms": 183.481, "p95_ms": 161.524}, "triton_wait": {"avg_ms": 65.092, "count": 215, "max_ms": 96.734, "p95_ms": 88.998}, "visualize": {"avg_ms": 4.759, "count": 215, "max_ms": 7.281, "p95_ms": 6.377}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.98, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2811}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5570.148, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x5587861ae340] error while decoding MB 17 9, bytestream -14", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 54, "h264_decode_error_count": 112, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T20:01:35+09:00", "last_error_summary": "[h264 @ 0x5587861ae340] error while decoding MB 17 9, bytestream -14", "last_exit_at": "2026-07-02T19:56:02+09:00", "last_returncode": 0, "last_stable_duration_sec": 333.125, "last_started_at": "2026-07-02T19:56:02+09:00", "pid": 15598, "process": {"exists": false, "pid": 15598}, "restart_count": 67, "restart_count_per_hour": 5, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 68}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 146.512, "sample_count": 2566, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 337335, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T20:01:34+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.058, "last_frame_pushed_at": "2026-07-02T20:01:35+09:00", "last_frame_read_age_sec": 0.05, "last_frame_read_at": "2026-07-02T20:01:35+09:00", "mjpeg": {"age_sec": 0.042, "exists": true, "mtime": "2026-07-02T20:01:35.114656+09:00", "mtime_epoch": 1782990095.1146564, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 163234}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T20:01:35+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1445.238, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 240, "pushed_frames": 240, "read_frames": 240, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.001, "pushed": 8.001, "read": 8.001}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.33, "count": 239, "max_ms": 9.9, "p95_ms": 7.881}, "gstreamer_push": {"avg_ms": 1.116, "count": 240, "max_ms": 1.503, "p95_ms": 1.33}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.39, "count": 239, "max_ms": 74.169, "p95_ms": 66.376}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.257, "count": 239, "max_ms": 103.85, "p95_ms": 88.266}, "yolo11_640_fire_11class": {"avg_ms": 51.103, "count": 239, "max_ms": 78.607, "p95_ms": 66.468}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.74, "count": 239, "max_ms": 88.542, "p95_ms": 79.814}}, "truncated_count": 0}, "processing_total": {"avg_ms": 114.059, "count": 240, "max_ms": 163.96, "p95_ms": 151.031}, "read_interval": {"avg_ms": 125.362, "count": 240, "max_ms": 323.058, "p95_ms": 161.952}, "triton_wait": {"avg_ms": 63.915, "count": 239, "max_ms": 103.862, "p95_ms": 88.56}, "visualize": {"avg_ms": 5.024, "count": 239, "max_ms": 7.946, "p95_ms": 6.295}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.996, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2811}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5576.246, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T20:01:35+09:00", "last_error_summary": "[h264 @ 0x5587861ae340] error while decoding MB 17 9, bytestream -14", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 54, "h264_decode_error_count": 112, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T20:01:35+09:00", "last_error_summary": "[h264 @ 0x5587861ae340] error while decoding MB 17 9, bytestream -14", "last_exit_at": "2026-07-02T20:01:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 333.199, "last_started_at": "2026-07-02T19:56:02+09:00", "pid": 15598, "process": {"exists": false, "pid": 15598}, "restart_count": 67, "restart_count_per_hour": 5, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 68}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 146.512, "sample_count": 2566, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 337336, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T20:01:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.006, "last_frame_pushed_at": "2026-07-02T20:01:35+09:00", "last_frame_read_age_sec": 0.124, "last_frame_read_at": "2026-07-02T20:01:35+09:00", "mjpeg": {"age_sec": 0.116, "exists": true, "mtime": "2026-07-02T20:01:35.114656+09:00", "mtime_epoch": 1782990095.1146564, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 163234}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T20:01:35+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1445.238, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 240, "pushed_frames": 240, "read_frames": 239, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.014, "pushed": 8.014, "read": 7.981}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.329, "count": 240, "max_ms": 9.9, "p95_ms": 7.813}, "gstreamer_push": {"avg_ms": 1.116, "count": 240, "max_ms": 1.503, "p95_ms": 1.33}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.416, "count": 240, "max_ms": 74.169, "p95_ms": 66.326}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.265, "count": 240, "max_ms": 103.85, "p95_ms": 86.903}, "yolo11_640_fire_11class": {"avg_ms": 51.13, "count": 240, "max_ms": 78.607, "p95_ms": 66.364}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.753, "count": 240, "max_ms": 88.542, "p95_ms": 79.631}}, "truncated_count": 0}, "processing_total": {"avg_ms": 113.956, "count": 240, "max_ms": 163.96, "p95_ms": 151.031}, "read_interval": {"avg_ms": 125.257, "count": 239, "max_ms": 323.058, "p95_ms": 163.168}, "triton_wait": {"avg_ms": 63.916, "count": 240, "max_ms": 103.862, "p95_ms": 88.278}, "visualize": {"avg_ms": 5.024, "count": 240, "max_ms": 7.946, "p95_ms": 6.28}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.948, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2811}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5578.863, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 54, "h264_decode_error_count": 112, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T20:01:35+09:00", "last_error_summary": "[h264 @ 0x5587861ae340] error while decoding MB 17 9, bytestream -14", "last_exit_at": "2026-07-02T20:01:35+09:00", "last_returncode": 0, "last_stable_duration_sec": 333.206, "last_started_at": "2026-07-02T19:56:02+09:00", "pid": 15598, "process": {"exists": false, "pid": 15598}, "restart_count": 67, "restart_count_per_hour": 5, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 68}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 146.512, "sample_count": 2566, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 337336, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T20:01:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["ffmpeg_restart_count_per_hour>=3", "h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.013, "last_frame_pushed_at": "2026-07-02T20:01:35+09:00", "last_frame_read_age_sec": 0.131, "last_frame_read_at": "2026-07-02T20:01:35+09:00", "mjpeg": {"age_sec": 0.122, "exists": true, "mtime": "2026-07-02T20:01:35.114656+09:00", "mtime_epoch": 1782990095.1146564, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 163234}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T20:01:35+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1445.238, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_warning", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 240, "pushed_frames": 240, "read_frames": 239, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.012, "pushed": 8.012, "read": 7.979}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.329, "count": 240, "max_ms": 9.9, "p95_ms": 7.813}, "gstreamer_push": {"avg_ms": 1.116, "count": 240, "max_ms": 1.503, "p95_ms": 1.33}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.416, "count": 240, "max_ms": 74.169, "p95_ms": 66.326}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.265, "count": 240, "max_ms": 103.85, "p95_ms": 86.903}, "yolo11_640_fire_11class": {"avg_ms": 51.13, "count": 240, "max_ms": 78.607, "p95_ms": 66.364}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.753, "count": 240, "max_ms": 88.542, "p95_ms": 79.631}}, "truncated_count": 0}, "processing_total": {"avg_ms": 113.956, "count": 240, "max_ms": 163.96, "p95_ms": 151.031}, "read_interval": {"avg_ms": 125.257, "count": 239, "max_ms": 323.058, "p95_ms": 163.168}, "triton_wait": {"avg_ms": 63.916, "count": 240, "max_ms": 103.862, "p95_ms": 88.278}, "visualize": {"avg_ms": 5.024, "count": 240, "max_ms": 7.946, "p95_ms": 6.28}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.954, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2811}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5687.102, "limit_mb": 8796093022207.996}, "detail": {"pid": 15705, "restart_count": 68, "start_count": 69}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 54, "h264_decode_error_count": 112, "h264_decode_error_count_per_hour": 15, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T20:01:35+09:00", "last_error_summary": "[h264 @ 0x5587861ae340] error while decoding MB 17 9, bytestream -14", "last_exit_at": "2026-07-02T20:01:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T20:01:37+09:00", "pid": 15705, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.125, "name": "ffmpeg", "pid": 15705, "rss_mb": 139.125, "state": "S (sleeping)", "threads": 106}, "restart_count": 68, "restart_count_per_hour": 6, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 69}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 146.72, "sample_count": 2529, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 337336, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T20:01:35+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.249, "last_frame_pushed_at": "2026-07-02T20:01:35+09:00", "last_frame_read_age_sec": 4.366, "last_frame_read_at": "2026-07-02T20:01:35+09:00", "mjpeg": {"age_sec": 4.23, "exists": true, "mtime": "2026-07-02T20:01:35.242659+09:00", "mtime_epoch": 1782990095.2426589, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 163352}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T20:01:39+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1445.238, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 211, "pushed_frames": 211, "read_frames": 210, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.052, "pushed": 7.052, "read": 7.019}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 6.219, "count": 211, "max_ms": 9.9, "p95_ms": 7.645}, "gstreamer_push": {"avg_ms": 1.129, "count": 211, "max_ms": 1.503, "p95_ms": 1.334}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.599, "count": 211, "max_ms": 74.169, "p95_ms": 66.376}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.736, "count": 211, "max_ms": 103.85, "p95_ms": 88.789}, "yolo11_640_fire_11class": {"avg_ms": 51.382, "count": 211, "max_ms": 78.607, "p95_ms": 66.644}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.992, "count": 211, "max_ms": 88.542, "p95_ms": 79.998}}, "truncated_count": 0}, "processing_total": {"avg_ms": 111.693, "count": 211, "max_ms": 163.96, "p95_ms": 147.178}, "read_interval": {"avg_ms": 122.31, "count": 210, "max_ms": 172.354, "p95_ms": 155.634}, "triton_wait": {"avg_ms": 64.449, "count": 211, "max_ms": 103.862, "p95_ms": 88.802}, "visualize": {"avg_ms": 4.899, "count": 211, "max_ms": 7.064, "p95_ms": 5.703}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.919, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2834}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5587.016, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x562c0bb36040] error while decoding MB 17 27, bytestream -22", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 55, "h264_decode_error_count": 113, "h264_decode_error_count_per_hour": 16, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T20:07:11+09:00", "last_error_summary": "[h264 @ 0x562c0bb36040] error while decoding MB 17 27, bytestream -22", "last_exit_at": "2026-07-02T20:01:37+09:00", "last_returncode": 0, "last_stable_duration_sec": 334.438, "last_started_at": "2026-07-02T20:01:37+09:00", "pid": 15705, "process": {"exists": false, "pid": 15705}, "restart_count": 68, "restart_count_per_hour": 6, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 69}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 146.131, "sample_count": 2548, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 340174, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T20:07:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.064, "last_frame_pushed_at": "2026-07-02T20:07:11+09:00", "last_frame_read_age_sec": 0.053, "last_frame_read_at": "2026-07-02T20:07:11+09:00", "mjpeg": {"age_sec": 0.045, "exists": true, "mtime": "2026-07-02T20:07:11.864882+09:00", "mtime_epoch": 1782990431.864882, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164126}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T20:07:11+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1446.527, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 262, "pushed_frames": 262, "read_frames": 262, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.737, "pushed": 8.737, "read": 8.737}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.676, "count": 262, "max_ms": 9.105, "p95_ms": 7.8}, "gstreamer_push": {"avg_ms": 1.175, "count": 262, "max_ms": 1.472, "p95_ms": 1.408}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.367, "count": 262, "max_ms": 72.447, "p95_ms": 65.223}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.322, "count": 262, "max_ms": 109.131, "p95_ms": 87.698}, "yolo11_640_fire_11class": {"avg_ms": 49.324, "count": 262, "max_ms": 67.206, "p95_ms": 64.027}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.988, "count": 262, "max_ms": 88.819, "p95_ms": 78.982}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.554, "count": 262, "max_ms": 192.105, "p95_ms": 156.408}, "read_interval": {"avg_ms": 114.673, "count": 262, "max_ms": 200.96, "p95_ms": 165.321}, "triton_wait": {"avg_ms": 64.049, "count": 262, "max_ms": 109.143, "p95_ms": 87.708}, "visualize": {"avg_ms": 4.744, "count": 262, "max_ms": 9.725, "p95_ms": 6.731}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.987, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2834}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5586.742, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T20:07:11+09:00", "last_error_summary": "[h264 @ 0x562c0bb36040] error while decoding MB 17 27, bytestream -22", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 55, "h264_decode_error_count": 113, "h264_decode_error_count_per_hour": 16, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T20:07:11+09:00", "last_error_summary": "[h264 @ 0x562c0bb36040] error while decoding MB 17 27, bytestream -22", "last_exit_at": "2026-07-02T20:07:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 334.467, "last_started_at": "2026-07-02T20:01:37+09:00", "pid": 15705, "process": {"exists": false, "pid": 15705}, "restart_count": 68, "restart_count_per_hour": 6, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 69}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 146.131, "sample_count": 2549, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 340175, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T20:07:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.008, "last_frame_pushed_at": "2026-07-02T20:07:11+09:00", "last_frame_read_age_sec": 0.082, "last_frame_read_at": "2026-07-02T20:07:11+09:00", "mjpeg": {"age_sec": 0.074, "exists": true, "mtime": "2026-07-02T20:07:11.864882+09:00", "mtime_epoch": 1782990431.864882, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164126}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T20:07:11+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1446.527, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 262, "pushed_frames": 262, "read_frames": 261, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.753, "pushed": 8.753, "read": 8.719}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.676, "count": 262, "max_ms": 9.105, "p95_ms": 7.8}, "gstreamer_push": {"avg_ms": 1.175, "count": 262, "max_ms": 1.472, "p95_ms": 1.408}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.306, "count": 262, "max_ms": 72.447, "p95_ms": 65.223}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.169, "count": 262, "max_ms": 109.131, "p95_ms": 87.404}, "yolo11_640_fire_11class": {"avg_ms": 49.264, "count": 262, "max_ms": 67.206, "p95_ms": 64.027}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.92, "count": 262, "max_ms": 88.819, "p95_ms": 78.982}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.406, "count": 262, "max_ms": 192.105, "p95_ms": 156.408}, "read_interval": {"avg_ms": 114.636, "count": 261, "max_ms": 200.96, "p95_ms": 165.321}, "triton_wait": {"avg_ms": 63.897, "count": 262, "max_ms": 109.143, "p95_ms": 87.416}, "visualize": {"avg_ms": 4.749, "count": 262, "max_ms": 9.725, "p95_ms": 6.731}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.933, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2834}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5586.57, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 55, "h264_decode_error_count": 113, "h264_decode_error_count_per_hour": 16, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T20:07:11+09:00", "last_error_summary": "[h264 @ 0x562c0bb36040] error while decoding MB 17 27, bytestream -22", "last_exit_at": "2026-07-02T20:07:11+09:00", "last_returncode": 0, "last_stable_duration_sec": 334.477, "last_started_at": "2026-07-02T20:01:37+09:00", "pid": 15705, "process": {"exists": false, "pid": 15705}, "restart_count": 68, "restart_count_per_hour": 6, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 69}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 146.131, "sample_count": 2549, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 340175, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T20:07:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.018, "last_frame_pushed_at": "2026-07-02T20:07:11+09:00", "last_frame_read_age_sec": 0.092, "last_frame_read_at": "2026-07-02T20:07:11+09:00", "mjpeg": {"age_sec": 0.084, "exists": true, "mtime": "2026-07-02T20:07:11.864882+09:00", "mtime_epoch": 1782990431.864882, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164126}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T20:07:11+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1446.527, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 262, "pushed_frames": 262, "read_frames": 261, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 8.75, "pushed": 8.75, "read": 8.717}, "hints": ["frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.676, "count": 262, "max_ms": 9.105, "p95_ms": 7.8}, "gstreamer_push": {"avg_ms": 1.175, "count": 262, "max_ms": 1.472, "p95_ms": 1.408}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.306, "count": 262, "max_ms": 72.447, "p95_ms": 65.223}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.169, "count": 262, "max_ms": 109.131, "p95_ms": 87.404}, "yolo11_640_fire_11class": {"avg_ms": 49.264, "count": 262, "max_ms": 67.206, "p95_ms": 64.027}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 57.92, "count": 262, "max_ms": 88.819, "p95_ms": 78.982}}, "truncated_count": 0}, "processing_total": {"avg_ms": 103.406, "count": 262, "max_ms": 192.105, "p95_ms": 156.408}, "read_interval": {"avg_ms": 114.636, "count": 261, "max_ms": 200.96, "p95_ms": 165.321}, "triton_wait": {"avg_ms": 63.897, "count": 262, "max_ms": 109.143, "p95_ms": 87.416}, "visualize": {"avg_ms": 4.749, "count": 262, "max_ms": 9.725, "p95_ms": 6.731}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.943, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 1, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2834}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5688.855, "limit_mb": 8796093022207.996}, "detail": {"pid": 15812, "restart_count": 69, "start_count": 70}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 55, "h264_decode_error_count": 113, "h264_decode_error_count_per_hour": 16, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T20:07:11+09:00", "last_error_summary": "[h264 @ 0x562c0bb36040] error while decoding MB 17 27, bytestream -22", "last_exit_at": "2026-07-02T20:07:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T20:07:14+09:00", "pid": 15812, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.359, "name": "ffmpeg", "pid": 15812, "rss_mb": 139.359, "state": "S (sleeping)", "threads": 106}, "restart_count": 69, "restart_count_per_hour": 7, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 70}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 146.131, "sample_count": 2511, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 340175, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T20:07:11+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.712, "last_frame_pushed_at": "2026-07-02T20:07:11+09:00", "last_frame_read_age_sec": 4.787, "last_frame_read_at": "2026-07-02T20:07:11+09:00", "mjpeg": {"age_sec": 4.698, "exists": true, "mtime": "2026-07-02T20:07:11.944883+09:00", "mtime_epoch": 1782990431.9448833, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 164265}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T20:07:16+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1446.527, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 219, "pushed_frames": 219, "read_frames": 218, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.305, "pushed": 7.305, "read": 7.271}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 5.709, "count": 218, "max_ms": 9.105, "p95_ms": 7.8}, "gstreamer_push": {"avg_ms": 1.171, "count": 219, "max_ms": 1.472, "p95_ms": 1.408}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 50.648, "count": 218, "max_ms": 72.447, "p95_ms": 65.316}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 62.557, "count": 218, "max_ms": 109.131, "p95_ms": 87.698}, "yolo11_640_fire_11class": {"avg_ms": 49.566, "count": 218, "max_ms": 67.206, "p95_ms": 64.027}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 58.334, "count": 218, "max_ms": 88.819, "p95_ms": 80.223}}, "truncated_count": 0}, "processing_total": {"avg_ms": 105.114, "count": 219, "max_ms": 192.105, "p95_ms": 159.78}, "read_interval": {"avg_ms": 116.325, "count": 218, "max_ms": 200.96, "p95_ms": 167.824}, "triton_wait": {"avg_ms": 64.375, "count": 218, "max_ms": 109.143, "p95_ms": 87.708}, "visualize": {"avg_ms": 4.804, "count": 218, "max_ms": 9.725, "p95_ms": 6.804}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.981, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2877}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5599.898, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 1, "rtsp_timeout_count": 0, "sample_message": "[h264 @ 0x55c302ea24c0] error while decoding MB 38 43, bytestream -11", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 56, "h264_decode_error_count": 114, "h264_decode_error_count_per_hour": 17, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T20:16:41+09:00", "last_error_summary": "[h264 @ 0x55c302ea24c0] error while decoding MB 38 43, bytestream -11", "last_exit_at": "2026-07-02T20:07:14+09:00", "last_returncode": 0, "last_stable_duration_sec": 566.366, "last_started_at": "2026-07-02T20:07:14+09:00", "pid": 15812, "process": {"exists": false, "pid": 15812}, "restart_count": 69, "restart_count_per_hour": 7, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 70}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 160.811, "sample_count": 2468, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 345291, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T20:16:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.046, "last_frame_pushed_at": "2026-07-02T20:16:40+09:00", "last_frame_read_age_sec": 0.039, "last_frame_read_at": "2026-07-02T20:16:40+09:00", "mjpeg": {"age_sec": 0.029, "exists": true, "mtime": "2026-07-02T20:16:40.979039+09:00", "mtime_epoch": 1782991000.9790394, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171921}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T20:16:41+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1449.105, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 214, "pushed_frames": 214, "read_frames": 214, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.139, "pushed": 7.139, "read": 7.139}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.332, "count": 213, "max_ms": 12.155, "p95_ms": 8.799}, "gstreamer_push": {"avg_ms": 1.056, "count": 214, "max_ms": 1.518, "p95_ms": 1.219}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.504, "count": 213, "max_ms": 75.361, "p95_ms": 68.928}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.32, "count": 213, "max_ms": 105.019, "p95_ms": 84.712}, "yolo11_640_fire_11class": {"avg_ms": 50.041, "count": 213, "max_ms": 75.351, "p95_ms": 68.068}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.703, "count": 213, "max_ms": 93.883, "p95_ms": 85.508}}, "truncated_count": 0}, "processing_total": {"avg_ms": 131.435, "count": 214, "max_ms": 200.78, "p95_ms": 167.352}, "read_interval": {"avg_ms": 140.583, "count": 214, "max_ms": 208.504, "p95_ms": 176.194}, "triton_wait": {"avg_ms": 63.938, "count": 213, "max_ms": 105.032, "p95_ms": 88.139}, "visualize": {"avg_ms": 5.865, "count": 213, "max_ms": 11.81, "p95_ms": 7.689}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.975, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2877}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5599.066, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T20:16:41+09:00", "last_error_summary": "[h264 @ 0x55c302ea24c0] error while decoding MB 38 43, bytestream -11", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 56, "h264_decode_error_count": 114, "h264_decode_error_count_per_hour": 17, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T20:16:41+09:00", "last_error_summary": "[h264 @ 0x55c302ea24c0] error while decoding MB 38 43, bytestream -11", "last_exit_at": "2026-07-02T20:16:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 566.44, "last_started_at": "2026-07-02T20:07:14+09:00", "pid": 15812, "process": {"exists": false, "pid": 15812}, "restart_count": 69, "restart_count_per_hour": 7, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 70}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 160.811, "sample_count": 2468, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 345292, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T20:16:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.005, "last_frame_pushed_at": "2026-07-02T20:16:41+09:00", "last_frame_read_age_sec": 0.113, "last_frame_read_at": "2026-07-02T20:16:40+09:00", "mjpeg": {"age_sec": 0.104, "exists": true, "mtime": "2026-07-02T20:16:40.979039+09:00", "mtime_epoch": 1782991000.9790394, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171921}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T20:16:41+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1449.105, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 214, "pushed_frames": 214, "read_frames": 213, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.15, "pushed": 7.15, "read": 7.117}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.329, "count": 214, "max_ms": 12.155, "p95_ms": 8.799}, "gstreamer_push": {"avg_ms": 1.055, "count": 214, "max_ms": 1.518, "p95_ms": 1.215}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.471, "count": 214, "max_ms": 75.361, "p95_ms": 68.928}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.291, "count": 214, "max_ms": 105.019, "p95_ms": 84.712}, "yolo11_640_fire_11class": {"avg_ms": 50.032, "count": 214, "max_ms": 75.351, "p95_ms": 68.068}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.663, "count": 214, "max_ms": 93.883, "p95_ms": 85.508}}, "truncated_count": 0}, "processing_total": {"avg_ms": 131.239, "count": 214, "max_ms": 200.78, "p95_ms": 167.352}, "read_interval": {"avg_ms": 140.5, "count": 213, "max_ms": 208.504, "p95_ms": 176.194}, "triton_wait": {"avg_ms": 63.898, "count": 214, "max_ms": 105.032, "p95_ms": 88.139}, "visualize": {"avg_ms": 5.864, "count": 214, "max_ms": 11.81, "p95_ms": 7.689}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.93, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2877}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5599.059, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 56, "h264_decode_error_count": 114, "h264_decode_error_count_per_hour": 17, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T20:16:41+09:00", "last_error_summary": "[h264 @ 0x55c302ea24c0] error while decoding MB 38 43, bytestream -11", "last_exit_at": "2026-07-02T20:16:41+09:00", "last_returncode": 0, "last_stable_duration_sec": 566.446, "last_started_at": "2026-07-02T20:07:14+09:00", "pid": 15812, "process": {"exists": false, "pid": 15812}, "restart_count": 69, "restart_count_per_hour": 7, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": false, "start_count": 70}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 160.811, "sample_count": 2468, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 345292, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T20:16:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.011, "last_frame_pushed_at": "2026-07-02T20:16:41+09:00", "last_frame_read_age_sec": 0.119, "last_frame_read_at": "2026-07-02T20:16:40+09:00", "mjpeg": {"age_sec": 0.109, "exists": true, "mtime": "2026-07-02T20:16:40.979039+09:00", "mtime_epoch": 1782991000.9790394, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171921}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T20:16:41+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1449.105, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 214, "pushed_frames": 214, "read_frames": 213, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 7.149, "pushed": 7.149, "read": 7.115}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.329, "count": 214, "max_ms": 12.155, "p95_ms": 8.799}, "gstreamer_push": {"avg_ms": 1.055, "count": 214, "max_ms": 1.518, "p95_ms": 1.215}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.471, "count": 214, "max_ms": 75.361, "p95_ms": 68.928}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 61.291, "count": 214, "max_ms": 105.019, "p95_ms": 84.712}, "yolo11_640_fire_11class": {"avg_ms": 50.032, "count": 214, "max_ms": 75.351, "p95_ms": 68.068}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 60.663, "count": 214, "max_ms": 93.883, "p95_ms": 85.508}}, "truncated_count": 0}, "processing_total": {"avg_ms": 131.239, "count": 214, "max_ms": 200.78, "p95_ms": 167.352}, "read_interval": {"avg_ms": 140.5, "count": 213, "max_ms": 208.504, "p95_ms": 176.194}, "triton_wait": {"avg_ms": 63.898, "count": 214, "max_ms": 105.032, "p95_ms": 88.139}, "visualize": {"avg_ms": 5.864, "count": 214, "max_ms": 11.81, "p95_ms": 7.689}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.935, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2877}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5705.113, "limit_mb": 8796093022207.996}, "detail": {"pid": 15919, "restart_count": 70, "start_count": 71}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 56, "h264_decode_error_count": 114, "h264_decode_error_count_per_hour": 17, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T20:16:41+09:00", "last_error_summary": "[h264 @ 0x55c302ea24c0] error while decoding MB 38 43, bytestream -11", "last_exit_at": "2026-07-02T20:16:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T20:16:43+09:00", "pid": 15919, "process": {"exists": true, "fd_count": 5, "hwm_mb": 138.543, "name": "ffmpeg", "pid": 15919, "rss_mb": 138.543, "state": "S (sleeping)", "threads": 106}, "restart_count": 70, "restart_count_per_hour": 8, "rtsp_timeout_count": 34, "rtsp_timeout_count_per_hour": 6, "rtsp_tolerant_mode": false, "running": true, "start_count": 71}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 160.957, "sample_count": 2420, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 345292, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T20:16:40+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.834, "last_frame_pushed_at": "2026-07-02T20:16:41+09:00", "last_frame_read_age_sec": 4.942, "last_frame_read_at": "2026-07-02T20:16:40+09:00", "mjpeg": {"age_sec": 4.816, "exists": true, "mtime": "2026-07-02T20:16:41.095042+09:00", "mtime_epoch": 1782991001.0950415, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 171842}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T20:16:45+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1449.105, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 182, "pushed_frames": 182, "read_frames": 181, "select_timeouts": 0, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 6.084, "pushed": 6.084, "read": 6.051}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant"], "latency_ms": {"analyze": {"avg_ms": 7.287, "count": 182, "max_ms": 12.155, "p95_ms": 8.76}, "gstreamer_push": {"avg_ms": 1.062, "count": 182, "max_ms": 1.518, "p95_ms": 1.204}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 51.018, "count": 182, "max_ms": 75.233, "p95_ms": 65.794}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 60.585, "count": 182, "max_ms": 100.915, "p95_ms": 80.076}, "yolo11_640_fire_11class": {"avg_ms": 49.311, "count": 182, "max_ms": 72.527, "p95_ms": 65.198}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 59.915, "count": 182, "max_ms": 92.783, "p95_ms": 80.754}}, "truncated_count": 0}, "processing_total": {"avg_ms": 129.275, "count": 182, "max_ms": 198.515, "p95_ms": 164.224}, "read_interval": {"avg_ms": 138.583, "count": 181, "max_ms": 206.176, "p95_ms": 173.166}, "triton_wait": {"avg_ms": 62.987, "count": 182, "max_ms": 100.928, "p95_ms": 84.784}, "visualize": {"avg_ms": 5.808, "count": 182, "max_ms": 11.81, "p95_ms": 7.584}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.913, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2896}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5598.363, "limit_mb": 8796093022207.996}, "detail": {"h264_decode_error_count": 11, "rtsp_timeout_count": 2, "sample_message": "[h264 @ 0x55d6c0ed1740] cabac decode of qscale diff failed at 24 34 [h264 @ 0x55d6c0ed1740] error while decoding MB 24 34, bytestream 136358 [h264 @ 0x55d6c0f2b240] left block unavailable for requested intra mode [h264 @ 0x55d6c0f2b240] error while decoding MB 0 6, bytestream 6620 [h264 @ 0x55d6c1acf380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d6c1acf380] error while decoding MB 0 40, bytestream 1184 [h264 @ 0x55d6c1acf380] cabac decode of qscale diff failed at 76 31 [h", "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_stderr_error", "ffmpeg": {"error_count": 57, "h264_decode_error_count": 125, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T20:21:47+09:00", "last_error_summary": "[h264 @ 0x55d6c0ed1740] cabac decode of qscale diff failed at 24 34 [h264 @ 0x55d6c0ed1740] error while decoding MB 24 34, bytestream 136358 [h264 @ 0x55d6c0f2b240] left block unavailable for requested intra mode [h264 @ 0x55d6c0f2b240] error while decoding MB 0 6, bytestream 6620 [h264 @ 0x55d6c1acf380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d6c1acf380] error while decoding MB 0 40, bytestream 1184 [h264 @ 0x55d6c1acf380] cabac decode of qscale diff failed at 76 31 [h", "last_exit_at": "2026-07-02T20:16:43+09:00", "last_returncode": 0, "last_stable_duration_sec": 303.209, "last_started_at": "2026-07-02T20:16:43+09:00", "pid": 15919, "process": {"exists": false, "pid": 15919}, "restart_count": 70, "restart_count_per_hour": 8, "rtsp_timeout_count": 36, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 71}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 141.484, "sample_count": 2283, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 347584, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T20:21:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.073, "last_frame_pushed_at": "2026-07-02T20:21:47+09:00", "last_frame_read_age_sec": 0.064, "last_frame_read_at": "2026-07-02T20:21:47+09:00", "mjpeg": {"age_sec": 0.058, "exists": true, "mtime": "2026-07-02T20:21:47.060417+09:00", "mtime_epoch": 1782991307.0604167, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 159076}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T20:21:47+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1450.137, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 88, "pushed_frames": 88, "read_frames": 88, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.936, "pushed": 2.936, "read": 2.936}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.397, "count": 88, "max_ms": 10.004, "p95_ms": 8.338}, "gstreamer_push": {"avg_ms": 1.067, "count": 88, "max_ms": 1.485, "p95_ms": 1.185}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.752, "count": 88, "max_ms": 72.599, "p95_ms": 66.636}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.647, "count": 88, "max_ms": 108.027, "p95_ms": 85.179}, "yolo11_640_fire_11class": {"avg_ms": 52.508, "count": 88, "max_ms": 76.074, "p95_ms": 66.408}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.098, "count": 88, "max_ms": 93.823, "p95_ms": 85.032}}, "truncated_count": 0}, "processing_total": {"avg_ms": 116.077, "count": 88, "max_ms": 152.33, "p95_ms": 137.703}, "read_interval": {"avg_ms": 341.199, "count": 88, "max_ms": 10127.761, "p95_ms": 149.914}, "triton_wait": {"avg_ms": 64.976, "count": 88, "max_ms": 108.035, "p95_ms": 88.797}, "visualize": {"avg_ms": 4.979, "count": 88, "max_ms": 7.429, "p95_ms": 5.561}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.972, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2896}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5597.918, "limit_mb": 8796093022207.996}, "detail": {"last_error_at": "2026-07-02T20:21:47+09:00", "last_error_summary": "[h264 @ 0x55d6c0ed1740] cabac decode of qscale diff failed at 24 34 [h264 @ 0x55d6c0ed1740] error while decoding MB 24 34, bytestream 136358 [h264 @ 0x55d6c0f2b240] left block unavailable for requested intra mode [h264 @ 0x55d6c0f2b240] error while decoding MB 0 6, bytestream 6620 [h264 @ 0x55d6c1acf380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d6c1acf380] error while decoding MB 0 40, bytestream 1184 [h264 @ 0x55d6c1acf380] cabac decode of qscale diff failed at 76 31 [h", "returncode": 0, "stderr_log_path": "/data/playlist/logs/live_runtime_health/ffmpeg/02.stderr.log"}, "event_type": "ffmpeg_terminated", "ffmpeg": {"error_count": 57, "h264_decode_error_count": 125, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T20:21:47+09:00", "last_error_summary": "[h264 @ 0x55d6c0ed1740] cabac decode of qscale diff failed at 24 34 [h264 @ 0x55d6c0ed1740] error while decoding MB 24 34, bytestream 136358 [h264 @ 0x55d6c0f2b240] left block unavailable for requested intra mode [h264 @ 0x55d6c0f2b240] error while decoding MB 0 6, bytestream 6620 [h264 @ 0x55d6c1acf380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d6c1acf380] error while decoding MB 0 40, bytestream 1184 [h264 @ 0x55d6c1acf380] cabac decode of qscale diff failed at 76 31 [h", "last_exit_at": "2026-07-02T20:21:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 303.26, "last_started_at": "2026-07-02T20:16:43+09:00", "pid": 15919, "process": {"exists": false, "pid": 15919}, "restart_count": 70, "restart_count_per_hour": 8, "rtsp_timeout_count": 36, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 71}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 141.484, "sample_count": 2284, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 347584, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T20:21:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.004, "last_frame_pushed_at": "2026-07-02T20:21:47+09:00", "last_frame_read_age_sec": 0.115, "last_frame_read_at": "2026-07-02T20:21:47+09:00", "mjpeg": {"age_sec": 0.11, "exists": true, "mtime": "2026-07-02T20:21:47.060417+09:00", "mtime_epoch": 1782991307.0604167, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 159076}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T20:21:47+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1450.137, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 88, "pushed_frames": 88, "read_frames": 87, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.943, "pushed": 2.943, "read": 2.909}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.397, "count": 88, "max_ms": 10.004, "p95_ms": 8.338}, "gstreamer_push": {"avg_ms": 1.067, "count": 88, "max_ms": 1.485, "p95_ms": 1.185}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.752, "count": 88, "max_ms": 72.599, "p95_ms": 66.636}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.647, "count": 88, "max_ms": 108.027, "p95_ms": 85.179}, "yolo11_640_fire_11class": {"avg_ms": 52.508, "count": 88, "max_ms": 76.074, "p95_ms": 66.408}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.098, "count": 88, "max_ms": 93.823, "p95_ms": 85.032}}, "truncated_count": 0}, "processing_total": {"avg_ms": 115.982, "count": 88, "max_ms": 152.33, "p95_ms": 137.703}, "read_interval": {"avg_ms": 343.63, "count": 87, "max_ms": 10127.761, "p95_ms": 149.914}, "triton_wait": {"avg_ms": 64.976, "count": 88, "max_ms": 108.035, "p95_ms": 88.797}, "visualize": {"avg_ms": 4.979, "count": 88, "max_ms": 7.429, "p95_ms": 5.561}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.906, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2896}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5597.91, "limit_mb": 8796093022207.996}, "event_type": "ffmpeg_not_running", "ffmpeg": {"error_count": 57, "h264_decode_error_count": 125, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T20:21:47+09:00", "last_error_summary": "[h264 @ 0x55d6c0ed1740] cabac decode of qscale diff failed at 24 34 [h264 @ 0x55d6c0ed1740] error while decoding MB 24 34, bytestream 136358 [h264 @ 0x55d6c0f2b240] left block unavailable for requested intra mode [h264 @ 0x55d6c0f2b240] error while decoding MB 0 6, bytestream 6620 [h264 @ 0x55d6c1acf380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d6c1acf380] error while decoding MB 0 40, bytestream 1184 [h264 @ 0x55d6c1acf380] cabac decode of qscale diff failed at 76 31 [h", "last_exit_at": "2026-07-02T20:21:47+09:00", "last_returncode": 0, "last_stable_duration_sec": 303.265, "last_started_at": "2026-07-02T20:16:43+09:00", "pid": 15919, "process": {"exists": false, "pid": 15919}, "restart_count": 70, "restart_count_per_hour": 8, "rtsp_timeout_count": 36, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": false, "start_count": 71}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 141.484, "sample_count": 2284, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 347584, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T20:21:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_not_running", "ffmpeg_restart_count_per_hour>=6"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10", "last_stable_duration_sec<600"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 0.01, "last_frame_pushed_at": "2026-07-02T20:21:47+09:00", "last_frame_read_age_sec": 0.12, "last_frame_read_at": "2026-07-02T20:21:47+09:00", "mjpeg": {"age_sec": 0.115, "exists": true, "mtime": "2026-07-02T20:21:47.060417+09:00", "mtime_epoch": 1782991307.0604167, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 159076}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T20:21:47+09:00", "process": {"exists": true, "fd_count": 119, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1450.137, "state": "S (sleeping)", "threads": 122}, "reason_codes": ["ffmpeg_not_running", "ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_warning", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 88, "pushed_frames": 88, "read_frames": 87, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 2.942, "pushed": 2.942, "read": 2.909}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.397, "count": 88, "max_ms": 10.004, "p95_ms": 8.338}, "gstreamer_push": {"avg_ms": 1.067, "count": 88, "max_ms": 1.485, "p95_ms": 1.185}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.752, "count": 88, "max_ms": 72.599, "p95_ms": 66.636}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 59.647, "count": 88, "max_ms": 108.027, "p95_ms": 85.179}, "yolo11_640_fire_11class": {"avg_ms": 52.508, "count": 88, "max_ms": 76.074, "p95_ms": 66.408}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 63.098, "count": 88, "max_ms": 93.823, "p95_ms": 85.032}}, "truncated_count": 0}, "processing_total": {"avg_ms": 115.982, "count": 88, "max_ms": 152.33, "p95_ms": 137.703}, "read_interval": {"avg_ms": 343.63, "count": 87, "max_ms": 10127.761, "p95_ms": 149.914}, "triton_wait": {"avg_ms": 64.976, "count": 88, "max_ms": 108.035, "p95_ms": 88.797}, "visualize": {"avg_ms": 4.979, "count": 88, "max_ms": 7.429, "p95_ms": 5.561}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.911, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}
{"analysis_logger": {"buffer_len": 0, "enabled": true, "file_path": "/data/playlist/logs/02/tracking_analysis_20260702_092040_b9bee411.jsonl", "flush_interval_seconds": 15.0, "max_buffer_records": 32, "record_counts": {"scenario_debug": 0, "scenario_event": 44, "tracking_summary": 2896}, "started": true}, "buffers": {"frame_buffer_len": 50, "frame_buffer_maxlen": 50, "frame_queue_maxsize": 10, "frame_queue_size": 0}, "cctv_id": "02", "cgroup_memory": {"current_mb": 5703.523, "limit_mb": 8796093022207.996}, "detail": {"pid": 16026, "restart_count": 71, "start_count": 72}, "event_type": "ffmpeg_started", "ffmpeg": {"error_count": 57, "h264_decode_error_count": 125, "h264_decode_error_count_per_hour": 28, "input_options": ["-rtsp_transport", "tcp", "-timeout", "10000000"], "last_error_at": "2026-07-02T20:21:47+09:00", "last_error_summary": "[h264 @ 0x55d6c0ed1740] cabac decode of qscale diff failed at 24 34 [h264 @ 0x55d6c0ed1740] error while decoding MB 24 34, bytestream 136358 [h264 @ 0x55d6c0f2b240] left block unavailable for requested intra mode [h264 @ 0x55d6c0f2b240] error while decoding MB 0 6, bytestream 6620 [h264 @ 0x55d6c1acf380] left block unavailable for requested intra4x4 mode -1 [h264 @ 0x55d6c1acf380] error while decoding MB 0 40, bytestream 1184 [h264 @ 0x55d6c1acf380] cabac decode of qscale diff failed at 76 31 [h", "last_exit_at": "2026-07-02T20:21:49+09:00", "last_returncode": 0, "last_stable_duration_sec": 2.001, "last_started_at": "2026-07-02T20:21:49+09:00", "pid": 16026, "process": {"exists": true, "fd_count": 5, "hwm_mb": 139.07, "name": "ffmpeg", "pid": 16026, "rss_mb": 139.07, "state": "S (sleeping)", "threads": 106}, "restart_count": 71, "restart_count_per_hour": 9, "rtsp_timeout_count": 36, "rtsp_timeout_count_per_hour": 8, "rtsp_tolerant_mode": false, "running": true, "start_count": 72}, "frame_flow": {"analysis": {"adaptive_enabled": false, "analysis_cap_enabled": false, "effective_analysis_fps": 10.0, "enabled": false, "last_decision": "initial", "processing_p95_ms": 141.411, "sample_count": 2252, "scenario_effective_fps_enabled": false, "target_fps": 10.0, "warmup_remaining_s": 0.0}, "queue_clear_age_s": 3.0, "queue_max_age_s": 2.0, "queue_maxsize": 10, "reader_queue_enabled": false, "stats": {"analysis_frames": 347584, "analysis_skipped": 0, "frames_dequeued": 0, "frames_queued": 0, "last_analysis_at": "2026-07-02T20:21:47+09:00", "last_queue_age_ms": null, "last_skip_at": "", "max_queue_age_ms": 0.0, "queue_cleared": 0, "queue_dropped": 0}}, "health": {"criticals": ["ffmpeg_restart_count_per_hour>=6", "last_stable_duration_sec<180"], "level": "critical", "thresholds": {"ffmpeg_restart_count_per_hour": {"critical": 6, "warning": 3}, "h264_decode_error_count_per_hour": {"critical": 50, "warning": 10}, "last_frame_pushed_age_sec": {"critical": 30, "warning": 10}, "last_frame_read_age_sec": {"critical": 30, "warning": 10}, "last_stable_duration_sec": {"critical_below": 180, "warning_below": 600}, "mjpeg_age_sec": {"critical": 30, "warning": 10}, "rtsp_timeout_count_per_hour": {"critical": 30, "warning": 10}}, "warnings": ["h264_decode_error_count_per_hour>=10"]}, "instance_id": "b9bee411", "last_frame_pushed_age_sec": 4.24, "last_frame_pushed_at": "2026-07-02T20:21:47+09:00", "last_frame_read_age_sec": 4.351, "last_frame_read_at": "2026-07-02T20:21:47+09:00", "mjpeg": {"age_sec": 4.226, "exists": true, "mtime": "2026-07-02T20:21:47.180419+09:00", "mtime_epoch": 1782991307.1804187, "path": "/data/playlist/streaming/02/mjpeg.jpg", "size_bytes": 159346}, "monitor_url": "http://192.168.50.244:1223/mjpeg/02", "observed_at": "2026-07-02T20:21:51+09:00", "process": {"exists": true, "fd_count": 120, "hwm_mb": 1498.547, "name": "celery", "pid": 237, "rss_mb": 1450.137, "state": "S (sleeping)", "threads": 123}, "reason_codes": ["ffmpeg_churn_critical", "ffmpeg_unstable_after_restart_critical", "h264_decode_error_warning"], "recommended_action": "channel_restart_recommended", "recorder": {"active_count": 0, "cooldown_sec": 100, "last_event_key_count": 1, "oldest_recorder_age_sec": null, "recorders": [], "running_ffmpeg_count": 0, "running_pids": [], "stopping_count": 0}, "recovery_eligible": true, "source_url": "rtsp://nk20001.iptime.org:554/live02", "stage_metrics": {"counts": {"inference_timeouts": 0, "invalid_reads": 0, "processed_frames": 54, "pushed_frames": 54, "read_frames": 54, "select_timeouts": 17, "triton_result_errors": 0}, "enabled": true, "fps": {"processed": 1.8, "pushed": 1.8, "read": 1.8}, "hints": ["read_fps_below_target", "processed_fps_below_target", "push_fps_below_target", "frame_processing_p95_over_budget", "triton_wait_p95_dominant", "ffmpeg_select_timeouts_seen"], "latency_ms": {"analyze": {"avg_ms": 6.706, "count": 54, "max_ms": 10.004, "p95_ms": 9.847}, "gstreamer_push": {"avg_ms": 1.089, "count": 54, "max_ms": 1.485, "p95_ms": 1.181}, "model_completion": {"model_count": 4, "models": {"yolo11-pose_640_pose_pretrained_by_Ultralytics_11s": {"avg_ms": 52.34, "count": 54, "max_ms": 70.366, "p95_ms": 66.636}, "yolo11_640_coco_Large_pretrained_by_Ultralytics_11": {"avg_ms": 58.77, "count": 54, "max_ms": 93.171, "p95_ms": 85.179}, "yolo11_640_fire_11class": {"avg_ms": 52.375, "count": 54, "max_ms": 76.074, "p95_ms": 70.915}, "yolo11m_640_worker_ppe_equipment_ops260616_28class": {"avg_ms": 62.479, "count": 54, "max_ms": 90.436, "p95_ms": 85.032}}, "truncated_count": 0}, "processing_total": {"avg_ms": 116.578, "count": 54, "max_ms": 143.822, "p95_ms": 138.159}, "read_interval": {"avg_ms": 477.113, "count": 54, "max_ms": 10127.761, "p95_ms": 156.264}, "triton_wait": {"avg_ms": 64.186, "count": 54, "max_ms": 93.183, "p95_ms": 88.797}, "visualize": {"avg_ms": 5.117, "count": 54, "max_ms": 6.603, "p95_ms": 5.998}}, "target_fps": 10.0, "target_frame_budget_ms": 100.0, "window_elapsed_s": 29.999, "window_s": 30.0}, "started_at": "2026-07-02T09:20:40+09:00", "state": "running", "task_id": "a94f6654-213a-4f19-a958-d42a3ac793ec", "trigger": "start"}